Sentiment-Analysis with word embeddings.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Sentiment-Analysis with word embeddings.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"nsWdzU-33p61","executionInfo":{"status":"ok","timestamp":1608902607975,"user_tz":-330,"elapsed":1138,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"6cc30b53-ad60-48a4-b303-4a3582048231"},"source":["from google.colab import drive\n","drive.mount('/gdrive')\n","%cd /gdrive/My Drive/Colab Notebooks/kaggle/Imdb movie reviews"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Drive already mounted at /gdrive; to attempt to forcibly remount, call drive.mount(\"/gdrive\", force_remount=True).\n","/gdrive/My Drive/Colab Notebooks/kaggle/Imdb movie reviews\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SlbgpVJ93kyn","executionInfo":{"status":"ok","timestamp":1608902610266,"user_tz":-330,"elapsed":3380,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["import numpy as np\n","import pandas as pd\n","\n","import tensorflow as tf\n","from tensorflow import keras\n","\n","from IPython.display import SVG\n","#from keras.utils import model_to_dot\n","from keras.utils.vis_utils import model_to_dot\n","from sklearn.metrics import classification_report,accuracy_score"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"lUH1PHEo3ky0","executionInfo":{"status":"ok","timestamp":1608902610697,"user_tz":-330,"elapsed":3793,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"30a4afe3-3add-4514-cf58-0e1dbb9d7552"},"source":["data=pd.read_csv(\"./data/Train.csv\")\n","test=pd.read_csv(\"./data/Test.csv\")\n","valid=pd.read_csv(\"./data/Valid.csv\")\n","print(data.info())\n","print(\"x\"*40)\n","print(test.info())"],"execution_count":3,"outputs":[{"output_type":"stream","text":["<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 40000 entries, 0 to 39999\n","Data columns (total 2 columns):\n"," #   Column  Non-Null Count  Dtype \n","---  ------  --------------  ----- \n"," 0   text    40000 non-null  object\n"," 1   label   40000 non-null  int64 \n","dtypes: int64(1), object(1)\n","memory usage: 625.1+ KB\n","None\n","xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\n","<class 'pandas.core.frame.DataFrame'>\n","RangeIndex: 5000 entries, 0 to 4999\n","Data columns (total 2 columns):\n"," #   Column  Non-Null Count  Dtype \n","---  ------  --------------  ----- \n"," 0   text    5000 non-null   object\n"," 1   label   5000 non-null   int64 \n","dtypes: int64(1), object(1)\n","memory usage: 78.2+ KB\n","None\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"t4poR5UW3ky9","executionInfo":{"status":"ok","timestamp":1608902610700,"user_tz":-330,"elapsed":3776,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"6813cb62-75c7-43d7-e91e-307498051c76"},"source":["data.label.value_counts()"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0    20019\n","1    19981\n","Name: label, dtype: int64"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BwdGaVzw54HO","executionInfo":{"status":"ok","timestamp":1608902638074,"user_tz":-330,"elapsed":31126,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"2068c311-6de0-4cce-c995-f37eaa9a6ba9"},"source":["words=[]\n","for sent in data.text:\n","  words.extend(sent.split())\n","\n","words_arr=np.array(words)\n","print(\"total no of uniques words \",np.unique(words_arr).shape)\n","print(np.unique(words_arr, return_counts=True))\n","del words_arr\n","del words"],"execution_count":5,"outputs":[{"output_type":"stream","text":["total no of uniques words  (381542,)\n","(array(['\\x10own', '!', '!!', ..., '₤250,000', '★★', '\\uf0b7'],\n","      dtype='<U78'), array([  1, 944,  67, ...,   1,   1,   2]))\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"lwxCvoTn_0lE","executionInfo":{"status":"ok","timestamp":1608902638081,"user_tz":-330,"elapsed":31110,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# data_path = keras.utils.get_file(\n","#     \"news20.tar.gz\",\n","#     \"http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.tar.gz\",\n","#     untar=True,\n","# )\n","# print(data_path)"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"xjh2cEKlAFIh","executionInfo":{"status":"ok","timestamp":1608902638086,"user_tz":-330,"elapsed":31098,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# import os\n","# import pathlib\n","\n","# data_dir = pathlib.Path(data_path).parent / \"20_newsgroup\"\n","# dirnames = os.listdir(data_dir)\n","# print(\"Number of directories:\", len(dirnames))\n","# print(\"Directory names:\", dirnames)\n","\n","# fnames = os.listdir(data_dir / \"comp.graphics\")\n","# print(\"Number of files in comp.graphics:\", len(fnames))\n","# print(\"Some example filenames:\", fnames[:5])"],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"Q0Rx-F_f7W4k","executionInfo":{"status":"ok","timestamp":1608903482477,"user_tz":-330,"elapsed":1095,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["MAX_NUM_WORDS=40000\n","MAX_SEQUENCE_LENGTH=200"],"execution_count":47,"outputs":[]},{"cell_type":"code","metadata":{"id":"bOrVe5e_vmF-","executionInfo":{"status":"ok","timestamp":1608903482860,"user_tz":-330,"elapsed":929,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["X=data.text\n","y=data.label\n","X_test=valid.text\n","y_test=valid.label"],"execution_count":48,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"6FgMpEbEKs62"},"source":["## Create a vocabulary index\n","\n","Let's use the `TextVectorization` to index the vocabulary found in the dataset.\n","Later, we'll use the same layer instance to vectorize the samples.\n","\n","Our layer will only consider the top 30,000 words, and will truncate or pad sequences to\n","be actually 200 tokens long."]},{"cell_type":"code","metadata":{"id":"XBeaZ7pQvlw2","executionInfo":{"status":"ok","timestamp":1608903489007,"user_tz":-330,"elapsed":6038,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n","\n","vectorizer = TextVectorization(max_tokens=MAX_NUM_WORDS, output_sequence_length=MAX_SEQUENCE_LENGTH)\n","text_ds = tf.data.Dataset.from_tensor_slices(X).batch(64)\n","vectorizer.adapt(text_ds)"],"execution_count":49,"outputs":[]},{"cell_type":"code","metadata":{"id":"AXPxL2Ls6B89","executionInfo":{"status":"ok","timestamp":1608903489009,"user_tz":-330,"elapsed":5361,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# print(len(list(text_ds)))\r\n","# print(list(text_ds)[1].shape)\r\n","# print(list(text_ds)[1])"],"execution_count":50,"outputs":[]},{"cell_type":"code","metadata":{"id":"hjTuJwSwPOhG","executionInfo":{"status":"ok","timestamp":1608903489010,"user_tz":-330,"elapsed":4748,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# dir(vectorizer)"],"execution_count":51,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dlT5wf0FvlnQ","executionInfo":{"status":"ok","timestamp":1608903489012,"user_tz":-330,"elapsed":4159,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"31852485-903a-4d09-dc4f-cb62408796bc"},"source":["vectorizer.get_vocabulary()[:5]"],"execution_count":52,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['', '[UNK]', 'the', 'and', 'a']"]},"metadata":{"tags":[]},"execution_count":52}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"kV7ZW5lHvlaQ","executionInfo":{"status":"ok","timestamp":1608903489014,"user_tz":-330,"elapsed":3501,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"5a986fb3-71e4-4764-c84e-94134c7a29fb"},"source":["output = vectorizer([[\"the cat sat on the mat\"]])\n","output.numpy()[0, :6]"],"execution_count":53,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([    2,  1191,  1711,    21,     2, 15465])"]},"metadata":{"tags":[]},"execution_count":53}]},{"cell_type":"markdown","metadata":{"id":"WviWQ9LULcHC"},"source":["As you can see, \"the\" gets represented as \"2\". Why not 0, given that \"the\" was the first word in the vocabulary? That's because index 0 is reserved for padding and index 1 is reserved for \"out of vocabulary\" tokens.\n","\n","Here's a dict mapping words to their indices:"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ppcoD3mzvlQn","executionInfo":{"status":"ok","timestamp":1608903494857,"user_tz":-330,"elapsed":1099,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"e0750a23-553e-4189-9e87-e6870475545d"},"source":["voc = vectorizer.get_vocabulary()\n","word_index = dict(zip(voc, range(len(voc))))\n","\n","\n","test_sen = [\"the\", \"cat\", \"sat\", \"on\", \"the\", \"mat\"]\n","[word_index[w] for w in test_sen]"],"execution_count":54,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[2, 1191, 1711, 21, 2, 15465]"]},"metadata":{"tags":[]},"execution_count":54}]},{"cell_type":"markdown","metadata":{"id":"Nv58JOlH8bXH"},"source":["## Load pre-trained word embeddings"]},{"cell_type":"code","metadata":{"id":"3sS2-d3-LnYz","executionInfo":{"status":"ok","timestamp":1608903496069,"user_tz":-330,"elapsed":1040,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# !wget http://nlp.stanford.edu/data/glove.6B.zip\n","# !unzip -q glove.6B.zip"],"execution_count":55,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Ddxr3wWbLm0y","executionInfo":{"status":"ok","timestamp":1608903503617,"user_tz":-330,"elapsed":8122,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"6f1d9520-065e-4f0e-fd34-a6a3f9bd75a6"},"source":["import os\n","path_to_glove_file = os.path.join(\n","    \"glove.6B.100d.txt\"\n",")\n","\n","embeddings_index = {}\n","with open(path_to_glove_file) as f:\n","    for line in f:\n","        word, coefs = line.split(maxsplit=1)\n","        coefs = np.fromstring(coefs, \"f\", sep=\" \")\n","        embeddings_index[word] = coefs\n","\n","print(\"Found %s word vectors.\" % len(embeddings_index))"],"execution_count":56,"outputs":[{"output_type":"stream","text":["Found 400000 word vectors.\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"my9W67YUATMC"},"source":["Now, let's prepare a corresponding embedding matrix that we can use in a Keras\n","`Embedding` layer. It's a simple NumPy matrix where entry at index `i` is the pre-trained\n","vector for the word of index `i` in our `vectorizer`'s vocabulary."]},{"cell_type":"code","metadata":{"id":"9QTyJML2Lmlh","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1608903503619,"user_tz":-330,"elapsed":6915,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"146b3e98-555d-4052-fd29-9b061915b047"},"source":["num_tokens = len(voc) + 2\n","embedding_dim = 100\n","hits = 0\n","misses = 0\n","\n","# Prepare embedding matrix\n","embedding_matrix = np.zeros((num_tokens, embedding_dim))\n","for word, i in word_index.items():\n","    embedding_vector = embeddings_index.get(word)\n","    if embedding_vector is not None:\n","        # Words not found in embedding index will be all-zeros.\n","        # This includes the representation for \"padding\" and \"OOV\"\n","        embedding_matrix[i] = embedding_vector\n","        hits += 1\n","    else:\n","        misses += 1\n","print(\"Converted %d words (%d misses)\" % (hits, misses))\n"],"execution_count":57,"outputs":[{"output_type":"stream","text":["Converted 34723 words (5277 misses)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"W4GXMSFzWdvv"},"source":["### Embedding layer"]},{"cell_type":"code","metadata":{"id":"RDVMdfWTh-ey","executionInfo":{"status":"ok","timestamp":1608903503626,"user_tz":-330,"elapsed":2590,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["from tensorflow.keras.layers import Embedding\r\n","\r\n","embedding_layer = Embedding(\r\n","    num_tokens,\r\n","    embedding_dim,\r\n","    embeddings_initializer=keras.initializers.Constant(embedding_matrix),\r\n","    trainable=True,\r\n",")"],"execution_count":58,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"KsPw1xYYWjtM"},"source":["## Build the model\r\n","\r\n","> A simple 1D convnet with global max pooling and a classifier at the end."]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"e75kb3xoh-PM","executionInfo":{"status":"ok","timestamp":1608903515048,"user_tz":-330,"elapsed":1127,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"9a52dc95-4956-4d2a-db06-65b802902385"},"source":["from tensorflow.keras import layers\r\n","\r\n","int_sequences_input = keras.Input(shape=(None,), dtype=\"int64\")\r\n","embedded_sequences = embedding_layer(int_sequences_input)\r\n","x = layers.Conv1D(16, 5, activation=\"relu\")(embedded_sequences)\r\n","x = layers.Dropout(0.3)(x)\r\n","x = layers.MaxPooling1D(5)(x)\r\n","x = layers.Conv1D(16, 5, activation=\"relu\")(x)\r\n","x = layers.Dropout(0.3)(x)\r\n","x = layers.MaxPooling1D(5)(x)\r\n","x = layers.Conv1D(16, 5, activation=\"relu\")(x)\r\n","x = layers.Dropout(0.5)(x)\r\n","x = layers.GlobalMaxPooling1D()(x)\r\n","x = layers.Dense(64, activation=\"relu\")(x)\r\n","x = layers.Dropout(0.5)(x)\r\n","preds = layers.Dense(2, activation=\"softmax\")(x)\r\n","model = keras.Model(int_sequences_input, preds)\r\n","model.summary()\r\n","SVG(model_to_dot(model).create(prog='dot', format='svg'))"],"execution_count":59,"outputs":[{"output_type":"stream","text":["Model: \"model_9\"\n","_________________________________________________________________\n","Layer (type)                 Output Shape              Param #   \n","=================================================================\n","input_10 (InputLayer)        [(None, None)]            0         \n","_________________________________________________________________\n","embedding_3 (Embedding)      (None, None, 100)         4000200   \n","_________________________________________________________________\n","conv1d_21 (Conv1D)           (None, None, 16)          8016      \n","_________________________________________________________________\n","dropout_21 (Dropout)         (None, None, 16)          0         \n","_________________________________________________________________\n","max_pooling1d_12 (MaxPooling (None, None, 16)          0         \n","_________________________________________________________________\n","conv1d_22 (Conv1D)           (None, None, 16)          1296      \n","_________________________________________________________________\n","dropout_22 (Dropout)         (None, None, 16)          0         \n","_________________________________________________________________\n","max_pooling1d_13 (MaxPooling (None, None, 16)          0         \n","_________________________________________________________________\n","conv1d_23 (Conv1D)           (None, None, 16)          1296      \n","_________________________________________________________________\n","dropout_23 (Dropout)         (None, None, 16)          0         \n","_________________________________________________________________\n","global_max_pooling1d_9 (Glob (None, 16)                0         \n","_________________________________________________________________\n","dense_18 (Dense)             (None, 64)                1088      \n","_________________________________________________________________\n","dropout_24 (Dropout)         (None, 64)                0         \n","_________________________________________________________________\n","dense_19 (Dense)             (None, 2)                 130       \n","=================================================================\n","Total params: 4,012,026\n","Trainable params: 4,012,026\n","Non-trainable params: 0\n","_________________________________________________________________\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<IPython.core.display.SVG object>"],"image/svg+xml":"<svg height=\"1325pt\" viewBox=\"0.00 0.00 312.00 994.00\" width=\"416pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g class=\"graph\" id=\"graph0\" transform=\"scale(1.3333 1.3333) rotate(0) translate(4 990)\">\n<title>G</title>\n<polygon fill=\"#ffffff\" points=\"-4,4 -4,-990 308,-990 308,4 -4,4\" stroke=\"transparent\"/>\n<!-- 139741470843792 -->\n<g class=\"node\" id=\"node1\">\n<title>139741470843792</title>\n<polygon fill=\"none\" points=\"82,-949.5 82,-985.5 222,-985.5 222,-949.5 82,-949.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-963.8\">input_10: InputLayer</text>\n</g>\n<!-- 139741470881832 -->\n<g class=\"node\" id=\"node2\">\n<title>139741470881832</title>\n<polygon fill=\"none\" points=\"66.5,-876.5 66.5,-912.5 237.5,-912.5 237.5,-876.5 66.5,-876.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-890.8\">embedding_3: Embedding</text>\n</g>\n<!-- 139741470843792&#45;&gt;139741470881832 -->\n<g class=\"edge\" id=\"edge1\">\n<title>139741470843792-&gt;139741470881832</title>\n<path d=\"M152,-949.4551C152,-941.3828 152,-931.6764 152,-922.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-922.5903 152,-912.5904 148.5001,-922.5904 155.5001,-922.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470880880 -->\n<g class=\"node\" id=\"node3\">\n<title>139741470880880</title>\n<polygon fill=\"none\" points=\"82,-803.5 82,-839.5 222,-839.5 222,-803.5 82,-803.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-817.8\">conv1d_21: Conv1D</text>\n</g>\n<!-- 139741470881832&#45;&gt;139741470880880 -->\n<g class=\"edge\" id=\"edge2\">\n<title>139741470881832-&gt;139741470880880</title>\n<path d=\"M152,-876.4551C152,-868.3828 152,-858.6764 152,-849.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-849.5903 152,-839.5904 148.5001,-849.5904 155.5001,-849.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470880600 -->\n<g class=\"node\" id=\"node4\">\n<title>139741470880600</title>\n<polygon fill=\"none\" points=\"81.5,-730.5 81.5,-766.5 222.5,-766.5 222.5,-730.5 81.5,-730.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-744.8\">dropout_21: Dropout</text>\n</g>\n<!-- 139741470880880&#45;&gt;139741470880600 -->\n<g class=\"edge\" id=\"edge3\">\n<title>139741470880880-&gt;139741470880600</title>\n<path d=\"M152,-803.4551C152,-795.3828 152,-785.6764 152,-776.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-776.5903 152,-766.5904 148.5001,-776.5904 155.5001,-776.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470391320 -->\n<g class=\"node\" id=\"node5\">\n<title>139741470391320</title>\n<polygon fill=\"none\" points=\"37.5,-657.5 37.5,-693.5 266.5,-693.5 266.5,-657.5 37.5,-657.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-671.8\">max_pooling1d_12: MaxPooling1D</text>\n</g>\n<!-- 139741470880600&#45;&gt;139741470391320 -->\n<g class=\"edge\" id=\"edge4\">\n<title>139741470880600-&gt;139741470391320</title>\n<path d=\"M152,-730.4551C152,-722.3828 152,-712.6764 152,-703.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-703.5903 152,-693.5904 148.5001,-703.5904 155.5001,-703.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470904104 -->\n<g class=\"node\" id=\"node6\">\n<title>139741470904104</title>\n<polygon fill=\"none\" points=\"82,-584.5 82,-620.5 222,-620.5 222,-584.5 82,-584.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-598.8\">conv1d_22: Conv1D</text>\n</g>\n<!-- 139741470391320&#45;&gt;139741470904104 -->\n<g class=\"edge\" id=\"edge5\">\n<title>139741470391320-&gt;139741470904104</title>\n<path d=\"M152,-657.4551C152,-649.3828 152,-639.6764 152,-630.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-630.5903 152,-620.5904 148.5001,-630.5904 155.5001,-630.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470416680 -->\n<g class=\"node\" id=\"node7\">\n<title>139741470416680</title>\n<polygon fill=\"none\" points=\"81.5,-511.5 81.5,-547.5 222.5,-547.5 222.5,-511.5 81.5,-511.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-525.8\">dropout_22: Dropout</text>\n</g>\n<!-- 139741470904104&#45;&gt;139741470416680 -->\n<g class=\"edge\" id=\"edge6\">\n<title>139741470904104-&gt;139741470416680</title>\n<path d=\"M152,-584.4551C152,-576.3828 152,-566.6764 152,-557.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-557.5903 152,-547.5904 148.5001,-557.5904 155.5001,-557.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470458768 -->\n<g class=\"node\" id=\"node8\">\n<title>139741470458768</title>\n<polygon fill=\"none\" points=\"37.5,-438.5 37.5,-474.5 266.5,-474.5 266.5,-438.5 37.5,-438.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-452.8\">max_pooling1d_13: MaxPooling1D</text>\n</g>\n<!-- 139741470416680&#45;&gt;139741470458768 -->\n<g class=\"edge\" id=\"edge7\">\n<title>139741470416680-&gt;139741470458768</title>\n<path d=\"M152,-511.4551C152,-503.3828 152,-493.6764 152,-484.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-484.5903 152,-474.5904 148.5001,-484.5904 155.5001,-484.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470435872 -->\n<g class=\"node\" id=\"node9\">\n<title>139741470435872</title>\n<polygon fill=\"none\" points=\"82,-365.5 82,-401.5 222,-401.5 222,-365.5 82,-365.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-379.8\">conv1d_23: Conv1D</text>\n</g>\n<!-- 139741470458768&#45;&gt;139741470435872 -->\n<g class=\"edge\" id=\"edge8\">\n<title>139741470458768-&gt;139741470435872</title>\n<path d=\"M152,-438.4551C152,-430.3828 152,-420.6764 152,-411.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-411.5903 152,-401.5904 148.5001,-411.5904 155.5001,-411.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470416568 -->\n<g class=\"node\" id=\"node10\">\n<title>139741470416568</title>\n<polygon fill=\"none\" points=\"81.5,-292.5 81.5,-328.5 222.5,-328.5 222.5,-292.5 81.5,-292.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-306.8\">dropout_23: Dropout</text>\n</g>\n<!-- 139741470435872&#45;&gt;139741470416568 -->\n<g class=\"edge\" id=\"edge9\">\n<title>139741470435872-&gt;139741470416568</title>\n<path d=\"M152,-365.4551C152,-357.3828 152,-347.6764 152,-338.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-338.5903 152,-328.5904 148.5001,-338.5904 155.5001,-338.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470484016 -->\n<g class=\"node\" id=\"node11\">\n<title>139741470484016</title>\n<polygon fill=\"none\" points=\"0,-219.5 0,-255.5 304,-255.5 304,-219.5 0,-219.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-233.8\">global_max_pooling1d_9: GlobalMaxPooling1D</text>\n</g>\n<!-- 139741470416568&#45;&gt;139741470484016 -->\n<g class=\"edge\" id=\"edge10\">\n<title>139741470416568-&gt;139741470484016</title>\n<path d=\"M152,-292.4551C152,-284.3828 152,-274.6764 152,-265.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-265.5903 152,-255.5904 148.5001,-265.5904 155.5001,-265.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470483568 -->\n<g class=\"node\" id=\"node12\">\n<title>139741470483568</title>\n<polygon fill=\"none\" points=\"95,-146.5 95,-182.5 209,-182.5 209,-146.5 95,-146.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-160.8\">dense_18: Dense</text>\n</g>\n<!-- 139741470484016&#45;&gt;139741470483568 -->\n<g class=\"edge\" id=\"edge11\">\n<title>139741470484016-&gt;139741470483568</title>\n<path d=\"M152,-219.4551C152,-211.3828 152,-201.6764 152,-192.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-192.5903 152,-182.5904 148.5001,-192.5904 155.5001,-192.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470483456 -->\n<g class=\"node\" id=\"node13\">\n<title>139741470483456</title>\n<polygon fill=\"none\" points=\"81.5,-73.5 81.5,-109.5 222.5,-109.5 222.5,-73.5 81.5,-73.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-87.8\">dropout_24: Dropout</text>\n</g>\n<!-- 139741470483568&#45;&gt;139741470483456 -->\n<g class=\"edge\" id=\"edge12\">\n<title>139741470483568-&gt;139741470483456</title>\n<path d=\"M152,-146.4551C152,-138.3828 152,-128.6764 152,-119.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-119.5903 152,-109.5904 148.5001,-119.5904 155.5001,-119.5903\" stroke=\"#000000\"/>\n</g>\n<!-- 139741470486032 -->\n<g class=\"node\" id=\"node14\">\n<title>139741470486032</title>\n<polygon fill=\"none\" points=\"95,-.5 95,-36.5 209,-36.5 209,-.5 95,-.5\" stroke=\"#000000\"/>\n<text fill=\"#000000\" font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"152\" y=\"-14.8\">dense_19: Dense</text>\n</g>\n<!-- 139741470483456&#45;&gt;139741470486032 -->\n<g class=\"edge\" id=\"edge13\">\n<title>139741470483456-&gt;139741470486032</title>\n<path d=\"M152,-73.4551C152,-65.3828 152,-55.6764 152,-46.6817\" fill=\"none\" stroke=\"#000000\"/>\n<polygon fill=\"#000000\" points=\"155.5001,-46.5903 152,-36.5904 148.5001,-46.5904 155.5001,-46.5903\" stroke=\"#000000\"/>\n</g>\n</g>\n</svg>"},"metadata":{"tags":[]},"execution_count":59}]},{"cell_type":"markdown","metadata":{"id":"ELCwYndGXQfz"},"source":["## Train the model\r\n","\r\n","First, convert our list-of-strings data to NumPy arrays of integer indices. The arrays\r\n","are right-padded."]},{"cell_type":"code","metadata":{"id":"9Ro44indh-EP","executionInfo":{"status":"ok","timestamp":1608903521523,"user_tz":-330,"elapsed":5690,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["x_train = vectorizer(np.array([[s] for s in X])).numpy()\r\n","x_val = vectorizer(np.array([[s] for s in X_test])).numpy()\r\n","\r\n","y_train = np.array(y)\r\n","y_val = np.array(y_test)"],"execution_count":60,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bD5F6bI3h958","executionInfo":{"status":"ok","timestamp":1608903703417,"user_tz":-330,"elapsed":186952,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"8fe5f6c2-73ab-41c7-e380-18eb73ef66b0"},"source":["model.compile(\r\n","    loss=\"sparse_categorical_crossentropy\", optimizer=\"rmsprop\", metrics=[\"acc\"]\r\n",")\r\n","model.fit(x_train, y_train, batch_size=64, epochs=15,validation_data=(x_val,y_val))"],"execution_count":61,"outputs":[{"output_type":"stream","text":["Epoch 1/15\n","625/625 [==============================] - 13s 20ms/step - loss: 0.6788 - acc: 0.5645 - val_loss: 0.5373 - val_acc: 0.7800\n","Epoch 2/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.4425 - acc: 0.8024 - val_loss: 0.4360 - val_acc: 0.8240\n","Epoch 3/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.3499 - acc: 0.8518 - val_loss: 0.4093 - val_acc: 0.8540\n","Epoch 4/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.3004 - acc: 0.8791 - val_loss: 0.3964 - val_acc: 0.8750\n","Epoch 5/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.2726 - acc: 0.8921 - val_loss: 0.3549 - val_acc: 0.8768\n","Epoch 6/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.2447 - acc: 0.9025 - val_loss: 0.3306 - val_acc: 0.8772\n","Epoch 7/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.2194 - acc: 0.9191 - val_loss: 0.3261 - val_acc: 0.8826\n","Epoch 8/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1972 - acc: 0.9255 - val_loss: 0.3180 - val_acc: 0.8846\n","Epoch 9/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1774 - acc: 0.9343 - val_loss: 0.3093 - val_acc: 0.8844\n","Epoch 10/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1619 - acc: 0.9426 - val_loss: 0.3029 - val_acc: 0.8794\n","Epoch 11/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1398 - acc: 0.9501 - val_loss: 0.3079 - val_acc: 0.8790\n","Epoch 12/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1319 - acc: 0.9542 - val_loss: 0.3131 - val_acc: 0.8826\n","Epoch 13/15\n","625/625 [==============================] - 12s 20ms/step - loss: 0.1168 - acc: 0.9608 - val_loss: 0.3259 - val_acc: 0.8738\n","Epoch 14/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.1099 - acc: 0.9624 - val_loss: 0.3571 - val_acc: 0.8654\n","Epoch 15/15\n","625/625 [==============================] - 12s 19ms/step - loss: 0.0960 - acc: 0.9684 - val_loss: 0.3363 - val_acc: 0.8726\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["<tensorflow.python.keras.callbacks.History at 0x7f1818adccc0>"]},"metadata":{"tags":[]},"execution_count":61}]},{"cell_type":"code","metadata":{"id":"fq7pCNqyahx0","executionInfo":{"status":"aborted","timestamp":1608902660423,"user_tz":-330,"elapsed":53137,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}}},"source":["# string_input = keras.Input(shape=(1,), dtype=\"string\")\r\n","# x = vectorizer(string_input)\r\n","# preds = model(x)\r\n","# end_to_end_model = keras.Model(string_input, preds)\r\n","\r\n","# probabilities = end_to_end_model.predict(\r\n","#     # [[\"this message is about computer graphics and 3D modeling\"]]\r\n","#     [[ test.text[2]]]\r\n","# )\r\n","\r\n","# np.argmax(probabilities[0]),test.label[2]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"y47hyJV9h9t9","executionInfo":{"status":"ok","timestamp":1608903719453,"user_tz":-330,"elapsed":8419,"user":{"displayName":"Pavan B","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GivBjv5wBxtRQDZe7R3HO58DjlpWCRg5hWdoGA=s64","userId":"17119499537978033662"}},"outputId":"eefe51c5-6a01-4ba8-e811-c717fec7ac7c"},"source":["string_input = keras.Input(shape=(None,), dtype=\"string\")\r\n","x = vectorizer(string_input)\r\n","preds = model(x)\r\n","end_to_end_model = keras.Model(string_input, preds)\r\n","\r\n","# train data\r\n","probabilities = end_to_end_model.predict(\r\n","    # [[\"this message is about computer graphics and 3D modeling\"]]\r\n","    data.text\r\n",")\r\n","\r\n","y_pred=[np.argmax(prob) for prob in probabilities]\r\n","print(\"testing training data\")\r\n","print(classification_report(data.label, y_pred))\r\n","\r\n","# test data\r\n","probabilities = end_to_end_model.predict(\r\n","    # [[\"this message is about computer graphics and 3D modeling\"]]\r\n","    test.text\r\n",")\r\n","\r\n","y_pred=[np.argmax(prob) for prob in probabilities]\r\n","print(\"testing test data\")\r\n","print(classification_report(test.label, y_pred))\r\n"],"execution_count":62,"outputs":[{"output_type":"stream","text":["testing training data\n","              precision    recall  f1-score   support\n","\n","           0       0.98      0.99      0.99     20019\n","           1       0.99      0.98      0.99     19981\n","\n","    accuracy                           0.99     40000\n","   macro avg       0.99      0.99      0.99     40000\n","weighted avg       0.99      0.99      0.99     40000\n","\n","testing test data\n","              precision    recall  f1-score   support\n","\n","           0       0.87      0.89      0.88      2495\n","           1       0.89      0.86      0.88      2505\n","\n","    accuracy                           0.88      5000\n","   macro avg       0.88      0.88      0.88      5000\n","weighted avg       0.88      0.88      0.88      5000\n","\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"ERlkd2K_Y0AP"},"source":["# xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"]}]}