SERC-tc-tempeval3-Causal.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"SERC-tc-tempeval3-Causal.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"112twhknsz-jspyhzht75wyzlla1iVz-R","authorship_tag":"ABX9TyOQ10skkxMSKyipqYQcX9xB"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"OftKjCb0I_ax"},"source":["import pickle\n","import numpy as np\n","from keras.preprocessing.text import Tokenizer\n","from keras.preprocessing.sequence import pad_sequences\n","from keras.models import Sequential\n","from keras.layers import *\n","from keras import Model\n","from sklearn.model_selection import train_test_split\n","from keras.callbacks import EarlyStopping\n","import keras.metrics\n","import tensorflow as tf\n","from sklearn.metrics import precision_recall_curve\n","from sklearn.metrics import classification_report\n","from torchtext import data\n","import pandas as pd\n","from sklearn.metrics import accuracy_score\n","from torch.utils.data import Dataset, DataLoader\n","from collections import Counter\n","from sklearn.utils import class_weight\n","from sklearn.metrics import precision_recall_fscore_support"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"D0RKdcDwM89p"},"source":["def read_data(filename):\n","  dfile = open(filename, 'rb')     \n","  data = pickle.load(dfile)\n","  dfile.close()\n","  return data"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"u0v3pGXqx9-v"},"source":["X_train, Y_train, labels_train = read_data('CausalTimeBank-TempEval/data_train_causaltb')\n","X_test, Y_test, labels_test = read_data('CausalTimeBank-TempEval/data_test_tempeval')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"JWFk4gRty0JE"},"source":["unique_tokens = read_data('CausalTimeBank-TempEval/unique_tokens_causaltb')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_f9wBcQ6K4ga"},"source":["MAX_NB_WORDS = 5000\n","MAX_SEQUENCE_LENGTH = 175\n","EMBEDDING_DIM = 300\n","\n","VAL_SIZE = 0.15"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Nf8b21JI9rHm"},"source":["unique_pos, unique_deps, unique_words = unique_tokens[0], unique_tokens[1], unique_tokens[2]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"-4y5t8H9YFwp"},"source":["tokenizer1 = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer1.fit_on_texts(unique_pos)\n","word_index1 = tokenizer1.word_index\n","\n","tokenizer2 = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer2.fit_on_texts(unique_words)\n","word_index2 = tokenizer2.word_index\n","\n","tokenizer3 = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer3.fit_on_texts(unique_deps)\n","word_index3 = tokenizer3.word_index"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1VI4lkZegeaG"},"source":["#train\n","seq1 = tokenizer1.texts_to_sequences(X_train[0])\n","seq11 = pad_sequences(seq1, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq2 = tokenizer2.texts_to_sequences(X_train[2])\n","seq12 = pad_sequences(seq2, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq3 = tokenizer3.texts_to_sequences(X_train[1])\n","seq13 = pad_sequences(seq3, maxlen=MAX_SEQUENCE_LENGTH)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Rt6u9ZX35rox"},"source":["# train and val\n","\n","nb_validation_samples = int(VAL_SIZE*seq11.shape[0])\n","\n","x_train1 = seq11[:-nb_validation_samples]\n","x_train2 = seq12[:-nb_validation_samples]\n","x_train3 = seq13[:-nb_validation_samples]\n","y_train = Y_train[:-nb_validation_samples]\n","lab_train = labels_train[:-nb_validation_samples]\n","\n","x_val1 = seq11[-nb_validation_samples:]\n","x_val2 = seq12[-nb_validation_samples:]\n","x_val3 = seq13[-nb_validation_samples:]\n","y_val = Y_train[-nb_validation_samples:]\n","lab_val = labels_train[-nb_validation_samples:]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"_KjXSkGEgeaG"},"source":["#test\n","\n","seq1 = tokenizer1.texts_to_sequences(X_test[0])\n","seq11_test = pad_sequences(seq1, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq2 = tokenizer2.texts_to_sequences(X_test[2])\n","seq12_test = pad_sequences(seq2, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq3 = tokenizer3.texts_to_sequences(X_test[1])\n","seq13_test = pad_sequences(seq3, maxlen=MAX_SEQUENCE_LENGTH)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PocyTpOdDd_H"},"source":["pos_vec = read_data('pos.vector')\n","dep_vec = read_data('deps.vector')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TwuH756retvz"},"source":["word_vec = {}\n","word_vec['PADDING'] = 300\n","f = open('glove.42B.300d.txt')\n","for line in f:\n","    values = line.split()\n","    word = values[0]\n","    coefs = np.asarray(values[1:], dtype='float32')\n","    word_vec[word.lower()] = line\n","f.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"si3ARA0yePC1"},"source":["# pos tags\n","\n","embedding_matrix1 = np.zeros((len(word_index1) + 1, 28))\n","for word, i in word_index1.items():\n","    embedding_vector = pos_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix1[i] = np.asarray(embedding_vector.split()[1:], dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"tThSjOoSkI_6"},"source":["#word vec\n","\n","embedding_matrix2 = np.zeros((len(word_index2) + 1, EMBEDDING_DIM))\n","for word, i in word_index2.items():\n","    embedding_vector = word_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix2[i] = np.asarray(embedding_vector.split()[1:], dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Iuj_DKsYekHV"},"source":["# deps vec\n","\n","embedding_matrix3 = np.zeros((len(word_index3) + 1, len(dep_vec['PADDING'])))\n","for word, i in word_index3.items():\n","    embedding_vector = dep_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix3[i] = np.asarray(embedding_vector, dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"G3zXkgojnbTd"},"source":["def get_class_weights(training_labels):\n","    class_weights = class_weight.compute_class_weight('balanced',np.unique(training_labels),training_labels)\n","    uni = list(np.unique(training_labels))\n","\n","    labelset = ['CLINK', 'CLINK-R', 'O']\n","\n","    weights = []\n","\n","    for i in labelset:\n","      try:\n","        idx = uni.index(i)\n","        weights.append(class_weights[idx])\n","      except:\n","        weights.append(0)\n","    return weights"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"hOTSKM-As78J"},"source":["\n","\n","# To Extract Causal Features"]},{"cell_type":"code","metadata":{"id":"VDQ7pYpEAd4t"},"source":["def defineModel(l1,l2,l3,l4,d1,out,d):\n","\n","    embedding_layer1 = Embedding(len(word_index2) + 1,EMBEDDING_DIM,weights=[embedding_matrix2],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer2 = Embedding(len(word_index1) + 1,28,weights=[embedding_matrix1],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer3 = Embedding(len(word_index3) + 1,77,weights=[embedding_matrix3],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","\n","    wi = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    wi2 = embedding_layer1(wi)\n","\n","    pi_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    pi2_sen = embedding_layer2(pi_sen)\n","\n","    di_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    di2_sen = embedding_layer3(di_sen)\n","\n","\n","    lstm1_sen = Bidirectional(LSTM(l1, activation='tanh', dropout=d, return_sequences=True), name = 'bid1causal_sen')(pi2_sen)  #  pos encoded features\n","    lstm2_sen = Bidirectional(LSTM(l2, activation='tanh', dropout=d, return_sequences=True), name= 'bid2causal_sen')(di2_sen)   #  dep features\n","    lstm3 = Bidirectional(LSTM(l4, activation='tanh', dropout=d+0.1, return_sequences=True), name = 'bid3causal')(wi2)  #  woed features\n","\n","    hid_sen = concatenate([lstm1_sen, lstm2_sen, lstm3])    \n","    \n","    lstm5 = Bidirectional(LSTM(l4, activation='tanh', dropout=d), name = 'bid3causallstm2_sen')(hid_sen)\n","\n","    yii = Dense(d1, activation='relu', name='dense1')(lstm5)\n","    yi = Dense(out, activation=\"softmax\", name='dense2')(yii)\n","    model = Model(inputs=[pi_sen,di_sen,wi],outputs=yi)\n","    return model\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KhwdUPSB9DN6"},"source":["def trainModel():\n","    num_classes = 3\n","\n","    epochs = 50\n","    batchsize = 64\n","    lr = 0.01\n","    file1 = 'CausalTimeBank-TempEval/chkpt/'\n","    \n","    out = num_classes\n","\n","    checkpoint_filepath = file1 + 'model_causal_ctb'\n","    training_data, training_labels, val_data,  val_labels = [x_train1,x_train3,x_train2],lab_train, [x_val1,x_val3,x_val2] , lab_val\n","    weights = get_class_weights(training_labels)\n","\n","    set_nodes = [32, 32, 64, 64, 32, 0.1]\n","    l1 = set_nodes[0]\n","    l2 = set_nodes[1]\n","    l3 = set_nodes[2]\n","    l4 = set_nodes[3]\n","    d1 = set_nodes[4]\n","    d = set_nodes[5]\n","    optimizer = tf.keras.optimizers.RMSprop(learning_rate=lr)\n","\n","    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)\n","    callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)\n","\n","    model = defineModel(l1,l2,l3,l4,d1,out,d)\n","    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'], loss_weights=weights)\n","    model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback, model_checkpoint_callback], verbose=0)\n","    model.load_weights(checkpoint_filepath)\n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"8SErGtgaA-bV"},"source":["model_causal = trainModel()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"8k9pdnhQ74Zc"},"source":["# To Extract Temporal Features\n"]},{"cell_type":"code","metadata":{"id":"NwAnbNdsUsR4"},"source":["X_train_temp, Y_train_temp, labels_train_temp = read_data('CausalTimeBank-TempEval/data_train_temporal_causaltb')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"b5xyaCSIUsR9"},"source":["unique_tokens_temp = read_data('CausalTimeBank-TempEval/unique_tokens_temporal_causaltb')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"YGCioDtrUsR-"},"source":["unique_pos_temp, unique_deps_temp, unique_words_temp = unique_tokens[0], unique_tokens[1], unique_tokens[2]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"dz7mR-XpUsR-"},"source":["tokenizer1_temp = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer1_temp.fit_on_texts(unique_pos_temp)\n","word_index1_temp = tokenizer1_temp.word_index\n","\n","tokenizer2_temp = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer2_temp.fit_on_texts(unique_words_temp)\n","word_index2_temp = tokenizer2_temp.word_index\n","\n","tokenizer3_temp = Tokenizer(num_words=MAX_NB_WORDS)\n","tokenizer3_temp.fit_on_texts(unique_deps_temp)\n","word_index3_temp = tokenizer3_temp.word_index"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Jr1p9kYZUsR_"},"source":["#train\n","seq1_temp = tokenizer1_temp.texts_to_sequences(X_train_temp[0])\n","seq11_temp = pad_sequences(seq1_temp, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq2_temp = tokenizer2_temp.texts_to_sequences(X_train_temp[2])\n","seq12_temp = pad_sequences(seq2_temp, maxlen=MAX_SEQUENCE_LENGTH)\n","\n","seq3_temp = tokenizer3_temp.texts_to_sequences(X_train_temp[1])\n","seq13_temp = pad_sequences(seq3_temp, maxlen=MAX_SEQUENCE_LENGTH)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"G1mim4upUsR_"},"source":["# train and val\n","\n","nb_validation_samples = int(VAL_SIZE*seq11_temp.shape[0])\n","\n","x_train1_temp = seq11_temp[:-nb_validation_samples]\n","x_train2_temp = seq12_temp[:-nb_validation_samples]\n","x_train3_temp = seq13_temp[:-nb_validation_samples]\n","y_train_temp = Y_train_temp[:-nb_validation_samples]\n","lab_train_temp = labels_train_temp[:-nb_validation_samples]\n","\n","x_val1_temp = seq11_temp[-nb_validation_samples:]\n","x_val2_temp = seq12_temp[-nb_validation_samples:]\n","x_val3_temp = seq13_temp[-nb_validation_samples:]\n","y_val_temp = Y_train_temp[-nb_validation_samples:]\n","lab_val_temp = labels_train_temp[-nb_validation_samples:]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2wiHyk6WUsSC"},"source":["# pos tags\n","\n","embedding_matrix1 = np.zeros((len(word_index1) + 1, 28))\n","for word, i in word_index1.items():\n","    embedding_vector = pos_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix1[i] = np.asarray(embedding_vector.split()[1:], dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OICC7NgpUsSD"},"source":["#word vec\n","\n","embedding_matrix2 = np.zeros((len(word_index2) + 1, EMBEDDING_DIM))\n","for word, i in word_index2.items():\n","    embedding_vector = word_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix2[i] = np.asarray(embedding_vector.split()[1:], dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"s-qsHI-oUsSD"},"source":["# deps vec\n","\n","embedding_matrix3 = np.zeros((len(word_index3) + 1, len(dep_vec['PADDING'])))\n","for word, i in word_index3.items():\n","    embedding_vector = dep_vec.get(word)\n","    if embedding_vector is not None:\n","        embedding_matrix3[i] = np.asarray(embedding_vector, dtype='float32')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mYKEq50rWvYX"},"source":["def get_class_weights(training_labels):\n","    class_weights = class_weight.compute_class_weight('balanced',np.unique(training_labels),training_labels)\n","    uni = list(np.unique(training_labels))\n","\n","    labelset = [ 'BEFORE', 'AFTER', 'SIMULTANEOUS', 'IBEFORE', 'IAFTER', 'IS_INCLUDED', 'INCLUDES', 'IDENTITY', 'BEGUN_BY', 'ENDED_BY',\n","    'BEGINS','ENDS','DURING','DURING_INV']\n","\n","    weights = []\n","\n","    for i in labelset:\n","      try:\n","        idx = uni.index(i)\n","        weights.append(class_weights[idx])\n","      except:\n","        weights.append(0)\n","    return weights"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ao6pohouWvYY"},"source":["def defineModel(l1,l2,l3,l4,d1,out,d):\n","\n","    embedding_layer1 = Embedding(len(word_index2) + 1,EMBEDDING_DIM,weights=[embedding_matrix2],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer2 = Embedding(len(word_index1) + 1,28,weights=[embedding_matrix1],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer3 = Embedding(len(word_index3) + 1,77,weights=[embedding_matrix3],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","\n","    wi = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    wi2 = embedding_layer1(wi)\n","\n","    pi_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    pi2_sen = embedding_layer2(pi_sen)\n","\n","    di_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    di2_sen = embedding_layer3(di_sen)\n","\n","    lstm1_sen = Bidirectional(LSTM(l1, activation='tanh', dropout=d, return_sequences=True), name = 'bid1temp_sen')(pi2_sen)  #  pos features\n","    lstm2_sen = Bidirectional(LSTM(l2, activation='tanh', dropout=d, return_sequences=True), name= 'bid2temp_sen')(di2_sen)   #  dep features\n","    lstm3 = Bidirectional(LSTM(l4, activation='tanh', dropout=d+0.1, return_sequences=True), name = 'bid3temp')(wi2)  #  woed features\n","\n","    hid_sen = concatenate([lstm1_sen, lstm2_sen, lstm3])    \n","    \n","    lstm5 = Bidirectional(LSTM(l4, activation='tanh', dropout=d), name = 'bid3templstm2_sen')(hid_sen)\n","\n","    yii = Dense(d1, activation='relu', name='dense1temp')(lstm5)\n","    yi = Dense(out, activation=\"softmax\", name='dense2temp')(yii)\n","    model = Model(inputs=[pi_sen,di_sen,wi],outputs=yi)\n","    return model\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bwAGRIPuYYEP"},"source":["def trainModel():\n","    num_classes = 14\n","\n","    epochs = 50\n","    batchsize = 64\n","    lr = 0.005\n","    file1 = 'CausalTimeBank-TempEval/chkpt/'\n","    \n","    out = num_classes\n","\n","    training_data, y_train, training_labels, val_data, y_val, val_labels = [x_train1_temp,x_train3_temp,x_train2_temp], y_train_temp, lab_train_temp, [x_val1_temp,x_val3_temp,x_val2_temp], y_val_temp, lab_val_temp\n","    weights = get_class_weights(training_labels)\n","\n","    set_nodes = [32, 32, 64, 64, 32, 0.3]\n","    l1 = set_nodes[0]\n","    l2 = set_nodes[1]\n","    l3 = set_nodes[2]\n","    l4 = set_nodes[3]\n","    d1 = set_nodes[4]\n","    d = set_nodes[5]\n","    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n","\n","    checkpoint_filepath = file1 + f'model_temp'   \n","    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)\n","    callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)\n","\n","    model = defineModel(l1,l2,l3,l4,d1,out,d)\n","    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'], loss_weights=weights)\n","    model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback, model_checkpoint_callback], verbose=0)\n","    model.load_weights(checkpoint_filepath)\n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XLtVHl2mZRSg"},"source":["model_temp = trainModel()"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"X-zXeXfB9FC9"},"source":["# Joint Model for Causal Relation Classification"]},{"cell_type":"code","metadata":{"id":"X5yzudh9q_QN"},"source":["def get_class_weights(training_labels):\n","    class_weights = class_weight.compute_class_weight('balanced',np.unique(training_labels),training_labels)\n","    uni = list(np.unique(training_labels))\n","\n","    labelset = ['CLINK', 'CLINK-R', 'O']\n","\n","    weights = []\n","\n","    for i in labelset:\n","      try:\n","        idx = uni.index(i)\n","        weights.append(class_weights[idx])\n","      except:\n","        weights.append(0)\n","    return weights"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rTjDbczugeaM"},"source":["def defineModel(l1,l2,l3,l4,d1,out,d):\n","\n","    embedding_layer1 = Embedding(len(word_index2) + 1,EMBEDDING_DIM,weights=[embedding_matrix2],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer2 = Embedding(len(word_index1) + 1,28,weights=[embedding_matrix1],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","    embedding_layer3 = Embedding(len(word_index3) + 1,77,weights=[embedding_matrix3],input_length=MAX_SEQUENCE_LENGTH,trainable=False)\n","\n","    wi = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    wi2 = embedding_layer1(wi)\n","\n","    pi_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    pi2_sen = embedding_layer2(pi_sen)\n","\n","    di_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n","    di2_sen = embedding_layer3(di_sen)\n","\n","    lstm1temp = Bidirectional(LSTM(l1, activation='tanh', dropout=d, return_sequences=True), name = 'bid1temp_sen')(pi2_sen) \n","    lstm1temp.trainable = False\n","    lstm2temp = Bidirectional(LSTM(l2, activation='tanh', dropout=d, return_sequences=True), name= 'bid2temp_sen')(di2_sen) \n","    lstm2temp.trainable = False\n","    lstm3temp = Bidirectional(LSTM(l4, activation='tanh', dropout=d+0.1, return_sequences=True), name = 'bid3temp')(wi2) \n","    lstm3temp.trainable = False\n","\n","    hid_temp = concatenate([lstm1temp, lstm2temp, lstm3temp])   \n","    \n","    lstm4temp = Bidirectional(LSTM(l4, activation='tanh', dropout=d), name = 'bid3templstm2_sen')(hid_temp)\n","    lstm4temp.trainable = False\n","\n","    lstm1causal = Bidirectional(LSTM(l1, activation='tanh', dropout=d, return_sequences=True), name = 'bid1causal_sen')(pi2_sen)\n","    lstm1causal.trainable = False\n","    lstm2causal = Bidirectional(LSTM(l2, activation='tanh', dropout=d, return_sequences=True), name= 'bid2causal_sen')(di2_sen)\n","    lstm2causal.trainable = False   \n","    lstm3causal = Bidirectional(LSTM(l3, activation='tanh', dropout=0.45, return_sequences=True), name = 'bid3causal')(wi2) \n","    lstm3causal.trainable = False \n","\n","\n","    hid_causal = concatenate([lstm1causal, lstm2causal, lstm3causal])   \n","\n","\n","    lstm4causal = Bidirectional(LSTM(l4, activation='tanh', dropout=d), name = 'bid3causallstm2_sen')(hid_causal)\n","    lstm4causal.trainable = False\n","\n","    merged_features = concatenate([lstm4temp, lstm4causal])\n","\n","    yii = Dense(d1, activation='relu', name='denselayer1')(merged_features)\n","    yi = Dense(out, activation=\"softmax\", name='denselayer2')(yii)\n","    model = Model(inputs=[pi_sen,di_sen,wi],outputs=yi)\n","    return model\n"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"p--9j0U7539o"},"source":["# Train"]},{"cell_type":"code","metadata":{"id":"3-gpmQwwC01T"},"source":["def trainModel():\n","    num_classes = 3\n","\n","    epochs = 50\n","    batchsize = 64\n","    lr = 0.005\n","    file1 = 'CausalTimeBank-TempEval/chkpt/'\n","    \n","    out = num_classes\n","\n","    training_data,training_labels, val_data, val_labels = [x_train1,x_train3,x_train2],lab_train, [x_val1,x_val3,x_val2] , lab_val\n","    weights = get_class_weights(training_labels)\n","\n","    set_nodes = [32, 32, 64, 64, 32, 0.1]\n","    l1 = set_nodes[0]\n","    l2 = set_nodes[1]\n","    l3 = set_nodes[2]\n","    l4 = set_nodes[3]\n","    d1 = set_nodes[4]\n","    d = set_nodes[5]\n","    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)\n","\n","    checkpoint_filepath = file1 + f'model'   \n","    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)\n","    callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)\n","\n","    model = defineModel(l1,l2,l3,l4,d1,out,d)\n","\n","    model.load_weights('CausalTimeBank-TempEval/model_causal.h5', by_name =True) # to extract causal features\n","    model.load_weights('CausalTimeBank-TempEval/model_temporal.h5', by_name =True) # to extract temporal features\n","\n","    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'], loss_weights=weights)\n","    model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback, model_checkpoint_callback], verbose=0)\n","    model.load_weights(checkpoint_filepath)\n","    return model"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"iOwBL2Xnr85n"},"source":["model = trainModel()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bsNXJ5XqDXa4"},"source":["batchsize = 64\n","lr = 0.005\n","epochs = 50\n","\n","training_data, training_labels, val_data, val_labels = [x_train1,x_train3,x_train2],lab_train, [x_val1,x_val3,x_val2] , lab_val\n","weights = get_class_weights(training_labels)\n","\n","file1 = 'TimeBank/chkpt/'\n","checkpoint_filepath = file1 + f'model'\n","optimizer =  tf.keras.optimizers.Adam(learning_rate=lr)\n","model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)\n","callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)\n","model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback,model_checkpoint_callback],verbose=0)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"i72DhaXgSnBf"},"source":["# Best Model Results"]},{"cell_type":"code","metadata":{"id":"xPWOaKQ9qSYM"},"source":["def format_report(report, scores):\n","  # [ 'CLINK' ,'CLINK-R', 'OTHER' ]\n","\n","  print(f\"              {'{0:>10}'.format('precision')} {'{0:>10}'.format('recall')} {'{0:>10}'.format('f1-score')}\")\n","  print(f\"       causes {'{0:>10}'.format(round(report['0']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['0']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['0']['f1-score']*100.0, 1))}\")\n","  print(f\"    caused by {'{0:>10}'.format(round(report['1']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['1']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['1']['f1-score']*100.0, 1))}\")\n","  print(\"\")\n","  print(f\"    micro avg {'{0:>10}'.format(round(scores[0]*100.0, 1))} {'{0:>10}'.format(round(scores[1]*100.0, 1))} {'{0:>10}'.format(round(scores[2]*100.0, 1))}\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"k9MDoxc1ShXm","executionInfo":{"status":"ok","timestamp":1626629698758,"user_tz":-330,"elapsed":10979,"user":{"displayName":"Sanjana Soni","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14Ggp-piu-0Zjxo8-Zn1GFpKyM8lmlY-IrpARTr0l=s64","userId":"12156689352855342736"}},"outputId":"46ebc455-90ae-44f5-bf95-2c876f5a14c0"},"source":["data_test = [seq11_test,seq13_test,seq12_test]\n","model = defineModel(32,32,64,64,32,3,0.1)\n","model.load_weights('CausalTimeBank-TempEval/model.h5', by_name=True)\n","classes = np.argmax(model.predict(x = data_test), axis=-1)\n","y_test_classes = Y_test.argmax(1)\n","y_pred_classes = classes\n","\n","report = classification_report(y_true=y_test_classes, y_pred=y_pred_classes, zero_division=0, output_dict=True, digits= 3, labels=[0,1,2,3,4,5,6,7,8,9,10,11,12,13])\n","scores = precision_recall_fscore_support(y_true=y_test_classes, y_pred=y_pred_classes, average='micro', labels=[0,1,2,3,4,5,6,7,8,9,10,11,12,13])\n","format_report(report, scores)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:5 out of the last 7 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fb491d0b830> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.\n","               precision     recall   f1-score\n","       causes       94.1       94.1       94.1\n","    caused by       88.9       88.9       88.9\n","\n","    micro avg       92.3       92.3       92.3\n"],"name":"stdout"}]}]}