HAN_of_elmoMinor(1).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "HAN_of_elmoMinor.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "Pv51hxXAWLqy",
        "ZnZd4aZwY8cM",
        "ETeG5NvTZBJI"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3dAw1xhlSZjk"
      },
      "source": [
        "# Elmo"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pv51hxXAWLqy"
      },
      "source": [
        "## Import Packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YWD2PY8jfudF",
        "outputId": "6f49582b-79bc-403d-b076-24e5f3013406",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install tensorflow==2.3\n",
        "!pip install \"tensorflow_hub>=0.6.0\"\n",
        "!pip3 install tensorflow_text==2.3"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow==2.3 in /usr/local/lib/python3.6/dist-packages (2.3.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (0.2.0)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (0.3.3)\n",
            "Requirement already satisfied: tensorboard<3,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (2.3.0)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (3.12.4)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.12.1)\n",
            "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (2.3.0)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.6.3)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (3.3.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.33.2)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.15.0)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.4.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (0.10.0)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.1.0)\n",
            "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.1.2)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (0.35.1)\n",
            "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (1.18.5)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.3) (2.10.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (2.23.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (0.4.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (3.3.3)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (1.17.2)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (50.3.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow==2.3) (1.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow==2.3) (2020.6.20)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow==2.3) (2.0.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3) (4.1.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3) (4.6)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3) (0.2.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow==2.3) (3.1.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow==2.3) (3.4.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3\"->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow==2.3) (0.4.8)\n",
            "Requirement already satisfied: tensorflow_hub>=0.6.0 in /usr/local/lib/python3.6/dist-packages (0.10.0)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_hub>=0.6.0) (1.18.5)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_hub>=0.6.0) (3.12.4)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow_hub>=0.6.0) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow_hub>=0.6.0) (50.3.2)\n",
            "Collecting tensorflow_text==2.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/28/b2/2dbd90b93913afd07e6101b8b84327c401c394e60141c1e98590038060b3/tensorflow_text-2.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6MB 4.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: tensorflow<2.4,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_text==2.3) (2.3.0)\n",
            "Requirement already satisfied: numpy<1.19.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.18.5)\n",
            "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.3.0)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.15.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.33.2)\n",
            "Requirement already satisfied: astunparse==1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.6.3)\n",
            "Requirement already satisfied: tensorboard<3,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.3.0)\n",
            "Requirement already satisfied: scipy==1.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.4.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.2.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.35.1)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.10.0)\n",
            "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.1.2)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.3.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.12.1)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.1.0)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.3.3)\n",
            "Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.12.4)\n",
            "Requirement already satisfied: h5py<2.11.0,>=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.10.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (50.3.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.7.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.23.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.17.2)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.4.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.3.3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.0.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.0.4)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (4.6)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (4.1.1)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (1.3.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (2.0.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.1.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<3,>=2.3.0->tensorflow<2.4,>=2.3.0->tensorflow_text==2.3) (3.4.0)\n",
            "Installing collected packages: tensorflow-text\n",
            "Successfully installed tensorflow-text-2.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E12entNbSBuj",
        "outputId": "8e16ca42-ad14-47c6-c9e9-c37607618f1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "#Load packages\n",
        "import tensorflow as tf \n",
        "from tensorflow.keras.preprocessing.text import Tokenizer \n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences \n",
        "from sklearn import metrics, preprocessing,model_selection \n",
        "from sklearn.metrics import accuracy_score \n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.layers import Input, Lambda, Dense \n",
        "from tensorflow.keras.models import Model \n",
        "import tensorflow.keras.backend as K \n",
        "import matplotlib.pyplot as pit \n",
        "import seaborn as sns\n",
        "import numpy as np \n",
        "import string \n",
        "import pandas as pd \n",
        "import re \n",
        "import spacy \n",
        "from nltk.corpus import stopwords \n",
        "from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS \n",
        "from spacy.lang.en import English \n",
        "spacy.load('en') \n",
        "parser = English()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.feature_extraction.stop_words module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.feature_extraction.text. Anything that cannot be imported from sklearn.feature_extraction.text is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pHthsbyGsVXL"
      },
      "source": [
        "K= tf.compat.v1.keras.backend.get_session()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LrP8jJaDQNql",
        "outputId": "8efd646b-23a5-4eb3-e0ce-7e811c98e933",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZnZd4aZwY8cM"
      },
      "source": [
        "##load Elmo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jE_okjTsZAhH",
        "outputId": "6d0320a3-40bb-45ec-c45f-c57991ec2414",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# get elmo from tensorflow hub \n",
        "import tensorflow_hub as hub \n",
        "import tensorflow as tf \n",
        "\n",
        "embed = hub.load('https://tfhub.dev/google/nnlm-en-dim128/1')\n",
        "embed.signatures['default'](tf.convert_to_tensor(['my text', 'batch']))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'default': <tf.Tensor: shape=(2, 128), dtype=float32, numpy=\n",
              " array([[ 0.06817301,  0.12772666,  0.04918525, -0.03550394, -0.04533052,\n",
              "          0.04868454,  0.00771219, -0.02631581, -0.16468817,  0.1887698 ,\n",
              "          0.01009648, -0.18591057,  0.05160484,  0.13291915,  0.01235215,\n",
              "         -0.00492161, -0.06701647, -0.01161232, -0.01099506,  0.21438217,\n",
              "          0.04877856, -0.10143851, -0.12461483, -0.00835736, -0.01975276,\n",
              "         -0.0518438 ,  0.0135941 ,  0.04147662, -0.14309719, -0.10322613,\n",
              "         -0.08090361, -0.01536523, -0.11944003, -0.04822929,  0.13167971,\n",
              "         -0.06830869, -0.04490855,  0.03839394,  0.0882614 ,  0.01578063,\n",
              "         -0.16467947, -0.09377561,  0.01380978, -0.04058216,  0.03929568,\n",
              "          0.09588426, -0.15572111, -0.15270787, -0.04349912, -0.0844278 ,\n",
              "          0.00472539,  0.01002582,  0.03455054,  0.0139303 , -0.11444485,\n",
              "         -0.07043104,  0.03757556, -0.00822732,  0.07750841, -0.02362585,\n",
              "          0.11321948, -0.02998928,  0.19748685,  0.16866075,  0.05818968,\n",
              "         -0.08720152,  0.02429257, -0.08923671,  0.04004676, -0.11712038,\n",
              "         -0.04423016,  0.00084125,  0.09887315, -0.10869319, -0.02063273,\n",
              "         -0.0078976 ,  0.03247038, -0.053636  , -0.06318824,  0.09211446,\n",
              "          0.06257474, -0.04052477, -0.02375481,  0.06692737,  0.01396037,\n",
              "         -0.00387887, -0.11007597, -0.02819042,  0.02156279, -0.03867229,\n",
              "          0.09522782,  0.03073994,  0.1777497 ,  0.07217226,  0.2552764 ,\n",
              "          0.00558206, -0.13917604, -0.01151034, -0.00699278, -0.00058617,\n",
              "         -0.12016475, -0.04845744,  0.00108416,  0.01308661, -0.08460812,\n",
              "          0.04146604, -0.02392333,  0.0211766 ,  0.01239127,  0.09751251,\n",
              "          0.08847063, -0.04759594, -0.14307027, -0.05373308,  0.06285188,\n",
              "         -0.06437098,  0.07693451, -0.10320926, -0.13538882,  0.02908451,\n",
              "         -0.01237198,  0.05219781, -0.02561041,  0.06948396,  0.05050331,\n",
              "         -0.0589957 ,  0.14204101,  0.05099661],\n",
              "        [ 0.01706688,  0.16787235,  0.09149991, -0.14499378, -0.08243445,\n",
              "          0.12357763,  0.01297838, -0.00048092,  0.0466181 ,  0.1043538 ,\n",
              "          0.02300307,  0.03475228, -0.04275631,  0.21015036, -0.02800887,\n",
              "         -0.05067778, -0.00960012, -0.01759498,  0.01835109, -0.07776673,\n",
              "         -0.03645713,  0.03972793, -0.03069391, -0.03422287, -0.16668513,\n",
              "         -0.04430129,  0.01050955, -0.04430391, -0.05041046,  0.01521919,\n",
              "         -0.00071418, -0.00097364,  0.19556542,  0.0457493 , -0.11583306,\n",
              "         -0.04204868,  0.04599042, -0.06314116, -0.10379948,  0.0802775 ,\n",
              "         -0.00482234, -0.02366483,  0.01422982, -0.1438511 ,  0.00452225,\n",
              "          0.00247407,  0.02109903,  0.05930688,  0.08789889, -0.09902695,\n",
              "          0.14681002, -0.09353893,  0.10695235,  0.00454453, -0.03457799,\n",
              "          0.05088221, -0.01846903, -0.10245369,  0.14374888, -0.10189284,\n",
              "          0.15101384, -0.00152795,  0.09687524,  0.06957269,  0.06327613,\n",
              "          0.03616752, -0.0828315 , -0.09079228, -0.01129711,  0.1017618 ,\n",
              "          0.0306166 , -0.01996683, -0.03714248, -0.00299824,  0.01350647,\n",
              "          0.00259201, -0.03050259,  0.01793307,  0.01617317,  0.07043888,\n",
              "          0.03073978, -0.02132967, -0.02019092,  0.02026299,  0.01462819,\n",
              "          0.01552583,  0.01370566, -0.11604404,  0.07736313,  0.00742614,\n",
              "         -0.19387761, -0.06397852, -0.00174941, -0.00899995, -0.00136152,\n",
              "         -0.12590362, -0.15958397,  0.07374638, -0.03661045,  0.18323964,\n",
              "          0.16642828, -0.01950032, -0.00106799, -0.20730937, -0.06857546,\n",
              "          0.11397488,  0.06187399,  0.22359262, -0.05364588, -0.14494267,\n",
              "         -0.03669563, -0.06828848, -0.12812346, -0.06980857,  0.24724436,\n",
              "         -0.07156715, -0.08053041, -0.06844967, -0.12146654,  0.1684804 ,\n",
              "          0.00748511, -0.18130283,  0.03431722, -0.14918843, -0.03616229,\n",
              "         -0.05937109, -0.00597944, -0.03736918]], dtype=float32)>}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ETeG5NvTZBJI"
      },
      "source": [
        "## Pre-process Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KTTh1xSIZIzR",
        "outputId": "6bf289a7-8c94-448f-8453-3688ceceda71",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Stop words and special characters \n",
        "import nltk \n",
        "nltk.download('stopwords') \n",
        "\n",
        "STOPLIST = set(stopwords.words('english') + list(ENGLISH_STOP_WORDS)) \n",
        "SYMBOLS = \" \".join(string.punctuation).split(\" \") + [\"-\",\"...\",\"”\",\"''\"]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwN6tjzCa7Hl"
      },
      "source": [
        "#Data Cleaner and tokenizer \n",
        "def tokenizeText(text): \n",
        "  text = text.strip().replace(\"\\n\", \" \").replace(\"\\r\", \" \") \n",
        "  text = text.lower() \n",
        "  \n",
        "  tokens = parser(text) \n",
        "  #lematization\n",
        "  lemmas = [] \n",
        "  for tok in tokens:\n",
        "     lemmas.append(tok.lemma_.lower().strip() if tok.lemma_ != \"-PRON-\" else tok.lower_)\n",
        "  tokens = lemmas \n",
        "  #reomve stop words and special charaters \n",
        "  tokens = [tok for tok in tokens if tok.lower() not in STOPLIST] \n",
        "  tokens = [tok for tok in tokens if tok not in SYMBOLS] \n",
        "  \n",
        "  tokens = [tok for tok in tokens if len(tok) >= 3]\n",
        "  \n",
        "  #remove remaining tokens that are not alphabetic \n",
        "  tokens = [tok for tok in tokens if tok.isalpha()] \n",
        "  \n",
        "  tokens = list(set(tokens)) \n",
        "  \n",
        "  return ' '.join(tokens[:]) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KtKqA7R2b3ih"
      },
      "source": [
        "def encode(le_enc, labels): \n",
        "  enc = le_enc.transform(labels) \n",
        "  return keras.utils.to_categorical(enc) \n",
        "\n",
        "def decode(le_enc, one_hot):\n",
        "   dec = np.argmax(one_hot, axis =1) \n",
        "   return le_enc.inverse_transform(dec) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9pw-LmCSOgc"
      },
      "source": [
        "# load the dataset \n",
        "import pandas as pd\n",
        "trainDF_Sheet_1 = pd.read_csv(\"/content/drive/My Drive/data/data_train.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DzdZNMeSSQHP",
        "outputId": "4505527b-7d98-4d98-acc9-5194ea6837be",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trainDF_Sheet_1.head(2) "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>review</th>\n",
              "      <th>is_bad_review</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>I am so angry that i made this post available...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>No Negative No real complaints the hotel was g...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              review  is_bad_review\n",
              "0   I am so angry that i made this post available...              1\n",
              "1  No Negative No real complaints the hotel was g...              0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u8adxxLUST8_",
        "outputId": "a25b996c-0229-4238-b941-8d2e640e3916",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trainDF_Sheet_1.shape "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(515738, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3X1XPGCYSUSs",
        "outputId": "c2851cca-39ca-41a8-f779-246b0515b321",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trainDF_Sheet_1['review'].unique() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([' I am so angry that i made this post available via all possible sites i use when planing my trips so no one will make the mistake of booking this place I made my booking via booking com We stayed for 6 nights in this hotel from 11 to 17 July Upon arrival we were placed in a small room on the 2nd floor of the hotel It turned out that this was not the room we booked I had specially reserved the 2 level duplex room so that we would have a big windows and high ceilings The room itself was ok if you don t mind the broken window that can not be closed hello rain and a mini fridge that contained some sort of a bio weapon at least i guessed so by the smell of it I intimately asked to change the room and after explaining 2 times that i booked a duplex btw it costs the same as a simple double but got way more volume due to the high ceiling was offered a room but only the next day SO i had to check out the next day before 11 o clock in order to get the room i waned to Not the best way to begin your holiday So we had to wait till 13 00 in order to check in my new room what a wonderful waist of my time The room 023 i got was just as i wanted to peaceful internal garden view big window We were tired from waiting the room so we placed our belongings and rushed to the city In the evening it turned out that there was a constant noise in the room i guess it was made by vibrating vent tubes or something it was constant and annoying as hell AND it did not stop even at 2 am making it hard to fall asleep for me and my wife I have an audio recording that i can not attach here but if you want i can send it via e mail The next day the technician came but was not able to determine the cause of the disturbing sound so i was offered to change the room once again the hotel was fully booked and they had only 1 room left the one that was smaller but seems newer  Only the park outside of the hotel was beautiful ',\n",
              "       'No Negative No real complaints the hotel was great great location surroundings rooms amenities and service Two recommendations however firstly the staff upon check in are very confusing regarding deposit payments and the staff offer you upon checkout to refund your original payment and you can make a new one Bit confusing Secondly the on site restaurant is a bit lacking very well thought out and excellent quality food for anyone of a vegetarian or vegan background but even a wrap or toasted sandwich option would be great Aside from those minor minor things fantastic spot and will be back when i return to Amsterdam ',\n",
              "       ' Rooms are nice but for elderly a bit difficult as most rooms are two story with narrow steps So ask for single level Inside the rooms are very very basic just tea coffee and boiler and no bar empty fridge  Location was good and staff were ok It is cute hotel the breakfast range is nice Will go back ',\n",
              "       ...,\n",
              "       ' The ac was useless It was a hot week in vienna and it only gave more hot airNo Positive',\n",
              "       'No Negative The rooms are enormous and really comfortable I believe that a family of 5 members could be more than comfy in these spaces ',\n",
              "       ' I was in 3rd floor It didn t work Free Wife  staff was very kind '],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TWwr5eOxSUpV",
        "outputId": "d8457250-b34d-4fc4-92f8-9242681b8e8c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "trainDF_Sheet_1['review'].value_counts() "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "No Negative Location                                                                                                                                                                                                                                                                                                    1009\n",
              " Nothing Everything                                                                                                                                                                                                                                                                                                      940\n",
              "No Negative Everything                                                                                                                                                                                                                                                                                                   593\n",
              "No Negative Great location                                                                                                                                                                                                                                                                                               256\n",
              "No Negative Everything                                                                                                                                                                                                                                                                                                   204\n",
              "                                                                                                                                                                                                                                                                                                                        ... \n",
              " Size of room very small decor tired some shortcomings in cleaning room  Location health spa                                                                                                                                                                                                                               1\n",
              " No fridge in room Restaurant Staff very friendly                                                                                                                                                                                                                                                                          1\n",
              " Like any other business hotel the place is a little cold the interiour like 100 others  Nice clean and practical hotel the staff is super friendly I would book it again anytime                                                                                                                                          1\n",
              " I asked for my bill to be mailed to me and still not arrived so now I have to chase it  Location and comfort                                                                                                                                                                                                              1\n",
              " Room is pretty small Although the bathroom is quite spacious the shower door and shower nozzle are rather problematic thus not a good showing experience Bad wifi signal not sure if it s because our room is on the edge  Wonderful location Less than 5 minute walk to Euston station Also has a bus stop nearby        1\n",
              "Name: review, Length: 499288, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pbzQwIIKTE4I"
      },
      "source": [
        "# model parameters\n",
        "\n",
        "vocab_size = 1000\n",
        "embedding_dim = 256\n",
        "max_length = 512\n",
        "trunc_type='post'\n",
        "padding_type='post'\n",
        "oov_tok = \"<00V>\"\n",
        "training_portion = .7"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ThxjnKWgTFT3"
      },
      "source": [
        "# train test split\n",
        "# ---------- \n",
        "              \n",
        "# proportion of training dataset\n",
        "train_size = int(trainDF_Sheet_1.shape[0] * training_portion)\n",
        "\n",
        "# training dataset\n",
        "train_sentences = trainDF_Sheet_1['review'][:train_size]\n",
        "train_labels = trainDF_Sheet_1['is_bad_review'][:train_size]\n",
        "\n",
        "# validation dataset\n",
        "validation_sentences = trainDF_Sheet_1['review'][train_size: ]\n",
        "validation_labels = trainDF_Sheet_1['is_bad_review'][train_size: ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bS_C32UnXnQF",
        "outputId": "f68039f1-d510-42dd-85cf-0fd398b70077",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from nltk.tokenize import word_tokenize\n",
        "nltk.download('punkt')\n",
        "def padd(sentence):\n",
        "    arr=word_tokenize(sentence)\n",
        "    for i in range(1000-len(arr)):\n",
        "        arr.append('null')\n",
        "    return arr[:1000]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w7MYGF5rbPeA"
      },
      "source": [
        "y_test = trainDF_Sheet_1[\"is_bad_review\"]\n",
        "train_sentences = trainDF_Sheet_1[\"review\"]\n",
        "train_data=[padd(each_sentence) for each_sentence in train_sentences[:1000]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9eGcYHAFcTdn"
      },
      "source": [
        "# for i in range(0,len(train_sentences),10000):\n",
        "training_embed=([embed.signatures['default'](tf.convert_to_tensor(each_sentence))['default'] for each_sentence in train_data])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fMpB3CJpNGoe",
        "outputId": "9de9638d-2c49-4065-d7b5-c2acb2459696",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "len(training_embed[0][0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "128"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UBr6i0LAaLtS"
      },
      "source": [
        "## With RMDL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AvzhneKdXAZZ",
        "outputId": "f51b6383-7afb-4788-8e34-6021b1e13186",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%cd My\\ Drive/han"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/han\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "roGq1hkwNeTn"
      },
      "source": [
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from tensorflow.keras.datasets import imdb\n",
        "from tensorflow.keras.preprocessing import sequence\n",
        "\n",
        "from han import HAN\n",
        "\n",
        "max_features = 5000\n",
        "maxlen_sentence = 16\n",
        "maxlen_word = 25\n",
        "batch_size = 32\n",
        "embedding_dims = 50\n",
        "epochs = 10"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nfQXTtHmaPF1",
        "outputId": "052b85b3-789e-4ac6-910d-04608d535b90",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 398
        }
      },
      "source": [
        "# model initialization\n",
        "model = HAN(maxlen_sentence, maxlen_word, max_features, embedding_dims)\n",
        "\n",
        "# compile model\n",
        "model.compile(loss='binary_crossentropy',\n",
        "    optimizer='adam',\n",
        "    metrics=['accuracy'])\n",
        "\n",
        "model.build(input_shape=[len(training_embed),1000,128] )\n",
        "\n",
        "early_stopping = EarlyStopping(monitor='val_accuracy', patience=3, mode='max')\n",
        "\n",
        "# model summary\n",
        "model. summary ()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-4b1460069ac4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     metrics=['accuracy'])\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtraining_embed\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m128\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0mearly_stopping\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_accuracy'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'max'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m    430\u001b[0m                            'method accepts an `inputs` argument.')\n\u001b[1;32m    431\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 432\u001b[0;31m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    433\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mInvalidArgumentError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m           raise ValueError('You cannot build your model by calling `build` '\n",
            "\u001b[0;32m/content/drive/My Drive/han/han.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'The rank of inputs of HAN must be 3, but now is %d'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmaxlen_sentence\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'get_shape'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rQcR6g6umpeG",
        "outputId": "1caabbc2-2654-46ad-d351-ccbd976ae5f8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "num_epochs = 8\n",
        "\n",
        "history = model.fit(np.array(training_embed), np.array(y_test[:1000]),\n",
        "   epochs=num_epochs,verbose=1)\n",
        "\n",
        "model.save_weights('/content/drive/My Drive/response-rmdl-elmo-model.h5')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/8\n",
            "32/32 [==============================] - 2s 67ms/step - loss: 0.3084 - accuracy: 0.9260\n",
            "Epoch 2/8\n",
            "32/32 [==============================] - 2s 65ms/step - loss: 0.1939 - accuracy: 0.9520\n",
            "Epoch 3/8\n",
            "32/32 [==============================] - 2s 65ms/step - loss: 0.1877 - accuracy: 0.9520\n",
            "Epoch 4/8\n",
            "32/32 [==============================] - 2s 66ms/step - loss: 0.1812 - accuracy: 0.9520\n",
            "Epoch 5/8\n",
            "32/32 [==============================] - 2s 65ms/step - loss: 0.1687 - accuracy: 0.9520\n",
            "Epoch 6/8\n",
            "32/32 [==============================] - 2s 66ms/step - loss: 0.1466 - accuracy: 0.9520\n",
            "Epoch 7/8\n",
            "32/32 [==============================] - 2s 65ms/step - loss: 0.1142 - accuracy: 0.9520\n",
            "Epoch 8/8\n",
            "32/32 [==============================] - 2s 65ms/step - loss: 0.0843 - accuracy: 0.9560\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZWcogO4LaQe"
      },
      "source": [
        "model.load_weights('/content/drive/My Drive/response-rmdl-elmo-model.h5')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qtl5wPAjLeQD"
      },
      "source": [
        "y_test = (y_test[:1000])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7BwNKtEQLo4P"
      },
      "source": [
        "prediction = model.predict(np.array(training_embed))\n",
        "y_pred = (prediction > 0.5)\n",
        "validation_sequences = tokenizer.texts_to_sequences(validation_sentences)\n",
        "validation_padded = pad_sequences(validation_sequences, padding=padding_type, maxlen=max_length)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AqMxXaOZmtH1"
      },
      "source": [
        "from sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix,accuracy_score\n",
        "print(\"F1-score: {0}\".format(f1_score(y_pred, y_test)))\n",
        "print(\"Precision: \",precision_score(y_pred, y_test))\n",
        "print(\"Recall: \",recall_score(y_pred, y_test))\n",
        "print(\"Accuracy: \",accuracy_score(y_pred,y_test))\n",
        "print(\"Confusion matrix: \")\n",
        "confusion_matrix(y_pred, y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fs_bqt8InQPZ"
      },
      "source": [
        "# accuracy and loss\n",
        "plt.figure(figsize=(10, 5))\n",
        "\n",
        "plt.subplot(1, 2, 1)\n",
        "plt.plot(history.history['accuracy'], label='Training Accuracy')\n",
        "plt.plot(history.history['val_accuracy'], label='Validation Accuracy')\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel('Accuracy')\n",
        "plt.legend()\n",
        "\n",
        "plt.subplot(1, 2, 2)\n",
        "plt.plot(history.history['loss'], label='Training Loss')\n",
        "plt.plot(history.history['val_loss'], label='Validation Loss')\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}