seminar_5.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lRC5Mk7DS7il"
      },
      "source": [
        "# CRF (and NN) for Named Entity Recognition\n",
        "\n",
        "### Based on https://github.com/aminaghoul/NER-PyTorch\n",
        "\n",
        "In this notebook we use conditional random fields (CRFs) which are generally used for named entity recognition in order to predict valid sequences.\n",
        "For example, one should not have a `B-PER` followed directly by another `B-PER`.\n",
        "\n",
        "For the CRF implementation, we will use the package [pytorch-crf](https://github.com/kmkurn/pytorch-crf)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install torchtext==0.6.0 pytorch-crf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nrs_X__dfOxM",
        "outputId": "c21f8be2-67a8-469f-9d3c-e873bb22a081"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchtext==0.6.0 in /usr/local/lib/python3.9/dist-packages (0.6.0)\n",
            "Requirement already satisfied: pytorch-crf in /usr/local/lib/python3.9/dist-packages (0.7.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (2.25.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (1.13.1+cu116)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (1.22.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (1.15.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (0.1.97)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.9/dist-packages (from torchtext==0.6.0) (4.65.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.6.0) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.6.0) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.6.0) (2.10)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests->torchtext==0.6.0) (4.0.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.9/dist-packages (from torch->torchtext==0.6.0) (4.5.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s7pqLoSXKbh6",
        "outputId": "9c6c9113-f63f-4a89-829b-2ceb0cf26905"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/cuda/__init__.py:497: UserWarning: Can't initialize NVML\n",
            "  warnings.warn(\"Can't initialize NVML\")\n"
          ]
        }
      ],
      "source": [
        "import time\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.optim import Adam\n",
        "from torchtext.data import Field, NestedField, BucketIterator\n",
        "from torchtext.datasets import SequenceTaggingDataset\n",
        "from torchtext.vocab import Vocab\n",
        "from collections import Counter\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "\n",
        "from torchtext import data\n",
        "from torchtext import datasets\n",
        "\n",
        "import spacy\n",
        "from torchcrf import CRF\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import time\n",
        "import random\n",
        "import string\n",
        "from itertools import chain"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k4LF_qyLKvfK"
      },
      "source": [
        "# Data preparation"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir data_ner"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1qePj_PLm1OS",
        "outputId": "700c3189-91dc-4d7c-cf23-9837bd7e76a6"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "mkdir: cannot create directory ‘data_ner’: File exists\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/aminaghoul/NER-PyTorch/raw/master/data/train.csv && mv train.csv data_ner/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h7ctBvmsm6T2",
        "outputId": "95148c92-f09b-41d4-9835-baa6bc158cda"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-03-17 03:29:03--  https://github.com/aminaghoul/NER-PyTorch/raw/master/data/train.csv\n",
            "Resolving github.com (github.com)... 20.27.177.113\n",
            "Connecting to github.com (github.com)|20.27.177.113|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/train.csv [following]\n",
            "--2023-03-17 03:29:04--  https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/train.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1666337 (1.6M) [text/plain]\n",
            "Saving to: ‘train.csv’\n",
            "\n",
            "train.csv           100%[===================>]   1.59M  --.-KB/s    in 0.07s   \n",
            "\n",
            "2023-03-17 03:29:05 (22.3 MB/s) - ‘train.csv’ saved [1666337/1666337]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/aminaghoul/NER-PyTorch/raw/master/data/valid.csv && mv valid.csv data_ner/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L57shyjEngHD",
        "outputId": "7245acfd-5a3d-4c14-e675-cc169821f61b"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-03-17 03:30:07--  https://github.com/aminaghoul/NER-PyTorch/raw/master/data/valid.csv\n",
            "Resolving github.com (github.com)... 20.27.177.113\n",
            "Connecting to github.com (github.com)|20.27.177.113|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/valid.csv [following]\n",
            "--2023-03-17 03:30:08--  https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/valid.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.111.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 419999 (410K) [text/plain]\n",
            "Saving to: ‘valid.csv’\n",
            "\n",
            "valid.csv           100%[===================>] 410.16K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2023-03-17 03:30:08 (9.40 MB/s) - ‘valid.csv’ saved [419999/419999]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!wget https://github.com/aminaghoul/NER-PyTorch/raw/master/data/test.csv && mv test.csv data_ner/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6s_2h9_HnrMa",
        "outputId": "7c557b21-d618-4056-fe5e-fc9e5c0a83ab"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-03-17 03:30:32--  https://github.com/aminaghoul/NER-PyTorch/raw/master/data/test.csv\n",
            "Resolving github.com (github.com)... 20.27.177.113\n",
            "Connecting to github.com (github.com)|20.27.177.113|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/test.csv [following]\n",
            "--2023-03-17 03:30:33--  https://raw.githubusercontent.com/aminaghoul/NER-PyTorch/master/data/test.csv\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.109.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 378113 (369K) [text/plain]\n",
            "Saving to: ‘test.csv’\n",
            "\n",
            "test.csv            100%[===================>] 369.25K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2023-03-17 03:30:33 (8.91 MB/s) - ‘test.csv’ saved [378113/378113]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# for the sake of reproducubility\n",
        "SEED = 1234\n",
        "\n",
        "random.seed(SEED)\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "metadata": {
        "id": "4Z1kjBd-3Dt0"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "JAamZSXiKnJl"
      },
      "outputs": [],
      "source": [
        "TEXT = data.Field(lower = False) \n",
        "TAG = data.Field(unk_token = None) # all the tags are known thus we set unk_token = None\n",
        "\n",
        "my_path = \"data_ner/\"\n",
        "train_data, valid_data, test_data = data.TabularDataset.splits(\n",
        "        path= my_path ,\n",
        "        train=\"train.csv\",\n",
        "        validation=\"valid.csv\",\n",
        "        test=\"test.csv\", format='csv', skip_header=True,\n",
        "        fields=((\"text\", TEXT), (\"tag\", TAG))\n",
        "    )\n",
        "MIN_FREQ = 0\n",
        "\n",
        "TEXT.build_vocab(train_data, \n",
        "                 min_freq = MIN_FREQ, # the words with frequency less than MIN_FREQ will be ignored and do not appear in the vocabulary\n",
        "                 vectors = \"glove.6B.100d\",\n",
        "                 unk_init = torch.Tensor.normal_)\n",
        "\n",
        "\n",
        "TAG.build_vocab(train_data)\n",
        "BATCH_SIZE = 16\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "#device = 'cpu'\n",
        "train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(\n",
        "    (train_data, valid_data, test_data), \n",
        "    batch_size = BATCH_SIZE,\n",
        "    device = device, sort=False)\n",
        "\n",
        "# padding index\n",
        "TEXT_PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]  \n",
        "TAG_PAD_IDX = TAG.vocab.stoi[TAG.pad_token]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qXY-SyrWKx4u"
      },
      "source": [
        "# Building the model\n",
        "\n",
        "For the model class, there are three main features:\n",
        "\n",
        "1. Prepare the CRF layer during initialization. We need to specify the number of possible tags in the text.\n",
        "2. include the logic of the CRF layer in the sequence `forward()`. There is a major change in the `forward()` due to the implementation of the `pytorch-crf` package. So far, we do forward propagation and loss calculation separately. Loss calculations are now integrated with forward propagation.\n",
        "3. Initialize all impossible transitions with a really low number (-100) in the `init_crf_transitions` function. This is where we impose the BIO sequence logic on the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "PW8shLquKyr-"
      },
      "outputs": [],
      "source": [
        "class NNCRF(nn.Module):\n",
        "    def __init__(self,\n",
        "                 input_dim,\n",
        "                 embedding_dim,\n",
        "                 hidden_dim,\n",
        "                 output_dim,\n",
        "                 emb_dropout,\n",
        "                 fc_dropout,\n",
        "                 word_pad_idx,\n",
        "                 tag_pad_idx):\n",
        "        super().__init__()\n",
        "        \n",
        "        # LAYER 1: Word Embedding\n",
        "        self.embedding_dim = embedding_dim\n",
        "        self.embedding = nn.Embedding(\n",
        "            num_embeddings=input_dim,\n",
        "            embedding_dim=embedding_dim,\n",
        "            padding_idx=word_pad_idx\n",
        "        )\n",
        "        self.emb_dropout = nn.Dropout(emb_dropout)\n",
        "\n",
        "        # LAYER 2: Fully-connected  \n",
        "        self.fc_dropout = nn.Dropout(fc_dropout)\n",
        "        self.fc = nn.Linear(embedding_dim, hidden_dim * 2) \n",
        "        \n",
        "        # LAYER 3: Fully-connected  \n",
        "        self.fc_dropout2 = nn.Dropout(fc_dropout)\n",
        "        self.fc2 = nn.Linear(hidden_dim * 2, output_dim) \n",
        "\n",
        "        # LAYER 4: CRF\n",
        "        self.tag_pad_idx = tag_pad_idx\n",
        "        self.crf = CRF(num_tags=output_dim)\n",
        "        \n",
        "        # init params with normal distribution\n",
        "        for name, param in self.named_parameters():\n",
        "            nn.init.normal_(param.data, mean=0, std=0.1)\n",
        "\n",
        "    def forward(self, words, tags=None):\n",
        "        # words = [sentence length, batch size]\n",
        "        # chars = [batch size, sentence length, word length)\n",
        "        # tags = [sentence length, batch size]\n",
        "        \n",
        "        # embedding_out = [sentence length, batch size, embedding dim]\n",
        "        embedding_out = self.emb_dropout(self.embedding(words))\n",
        "\n",
        "        # fc_out = [sentence length, batch size, hidden dim * 2]\n",
        "        fc_out = self.fc(self.fc_dropout(embedding_out))\n",
        "        \n",
        "        # fc_out2 = [sentence length, batch size, output dim]\n",
        "        fc_out2 = self.fc2(self.fc_dropout2(fc_out))\n",
        "      \n",
        "        if tags is not None:\n",
        "            mask = tags != self.tag_pad_idx\n",
        "            crf_out = self.crf.decode(fc_out2, mask=mask)\n",
        "            crf_loss = -self.crf(fc_out2, tags=tags, mask=mask) \n",
        "        else:\n",
        "            crf_out = self.crf.decode(fc_out2)\n",
        "            crf_loss = None\n",
        "          \n",
        "        return crf_out , crf_loss\n",
        "\n",
        "   \n",
        "  \n",
        "    def init_crf_transitions(self, tag_names, imp_value=-100):\n",
        "        num_tags = len(tag_names)\n",
        "        for i in range(num_tags):\n",
        "            tag_name = tag_names[i]\n",
        "            # I and <pad> cannot be in the beginning\n",
        "            if tag_name[0] == \"I\" or tag_name == \"<pad>\":\n",
        "                torch.nn.init.constant_(self.crf.start_transitions[i], imp_value)\n",
        "        # O - I transition is impossible\n",
        "        tag_is = {}\n",
        "        for tag_position in (\"B\", \"I\", \"O\"):\n",
        "            tag_is[tag_position] = [i for i, tag in enumerate(tag_names) if tag[0] == tag_position]\n",
        "        impossible_transitions_position = {\n",
        "            \"O\": \"I\"\n",
        "\n",
        "        }\n",
        "        for from_tag, to_tag_list in impossible_transitions_position.items():\n",
        "            to_tags = list(to_tag_list)\n",
        "            \n",
        "            for from_tag_i in tag_is[from_tag]:\n",
        "                for to_tag in to_tags:\n",
        "                    for to_tag_i in tag_is[to_tag]:\n",
        "                    \n",
        "                        torch.nn.init.constant_(\n",
        "                            self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
        "                        )\n",
        "        # impossible transitions between different types\n",
        "        impossible_transitions_tags = {\n",
        "            \"B\": \"I\",\n",
        "            \"I\": \"I\"\n",
        "        }\n",
        "        for from_tag, to_tag_list in impossible_transitions_tags.items():\n",
        "            to_tags = list(to_tag_list)\n",
        "            for from_tag_i in tag_is[from_tag]:\n",
        "                for to_tag in to_tags:\n",
        "                    for to_tag_i in tag_is[to_tag]:\n",
        "                        if tag_names[from_tag_i].split(\"-\")[1] != tag_names[to_tag_i].split(\"-\")[1]:\n",
        "                            torch.nn.init.constant_(\n",
        "                                self.crf.transitions[from_tag_i, to_tag_i], imp_value\n",
        "                            )\n",
        "\n",
        "    def count_parameters(self):\n",
        "        return sum(p.numel() for p in self.parameters() if p.requires_grad)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zb03iYctNAUM",
        "outputId": "be425477-f9b5-4c8f-8193-395c7cf1e6bd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The model has 2,419,562 trainable parameters.\n"
          ]
        }
      ],
      "source": [
        "embedding_dim=100\n",
        "tag_pad_idx=TAG_PAD_IDX\n",
        "model = NNCRF(\n",
        "    input_dim=len(TEXT.vocab),\n",
        "    embedding_dim=100,\n",
        "    hidden_dim=256,\n",
        "    output_dim=len(TAG.vocab),\n",
        "    emb_dropout=0.1,\n",
        "    fc_dropout=0.1,\n",
        "    word_pad_idx=TEXT_PAD_IDX,\n",
        "    tag_pad_idx=TAG_PAD_IDX\n",
        ")\n",
        "def init_weights(m):\n",
        "    for name, param in m.named_parameters():\n",
        "        nn.init.normal_(param.data, mean = 0, std = 0.1)\n",
        "        \n",
        "model.apply(init_weights)\n",
        "\n",
        "pretrained_embeddings = TEXT.vocab.vectors\n",
        "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
        "model.embedding.weight.data[tag_pad_idx] = torch.zeros(embedding_dim)\n",
        "\n",
        "# CRF transitions initialisation\n",
        "model.init_crf_transitions(\n",
        "    tag_names=TAG.vocab.itos\n",
        ")\n",
        "print(f\"The model has {model.count_parameters():,} trainable parameters.\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lGsjeyHlyWhx"
      },
      "source": [
        "We can access the transition matrix and make sure that the initialization is done as expected:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wfe5T7dlrS2j",
        "outputId": "af1045a2-4c9f-4085-b59d-f20985e2c2a2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Start and end tag transitions:\n",
            "TAG   \tSTART\tEND\n",
            "<pad> \t-100.0\t-0.12\n",
            "O     \t-0.05\t0.03\n",
            "B-LOC \t-0.07\t-0.02\n",
            "B-PER \t0.07\t0.13\n",
            "B-ORG \t0.0\t0.06\n",
            "I-PER \t-100.0\t-0.13\n",
            "I-ORG \t-100.0\t-0.06\n",
            "B-MISC\t0.04\t-0.13\n",
            "I-LOC \t-100.0\t0.06\n",
            "I-MISC\t-100.0\t-0.04\n",
            "\n",
            "Between tags transitions:\n",
            "   TO\tO    \tB-PER\tI-PER\n",
            "FROM\n",
            "O    \t0.01 \t-0.04\t-100.0\n",
            "B-PER\t-0.16\t0.1  \t-0.11\n",
            "I-PER\t0.12 \t0.01 \t-0.02\n"
          ]
        }
      ],
      "source": [
        "def print_crf_transitions(c, m):\n",
        "    tags = TAG.vocab.itos\n",
        "    max_len_tag = max([len(tag) for tag in tags])\n",
        "    print(\"Start and end tag transitions:\")\n",
        "    print(f\"{'TAG'.ljust(max_len_tag)}\\tSTART\\tEND\")\n",
        "    for tag, start_prob, end_prob in zip(tags, m.crf.start_transitions.tolist(), m.crf.end_transitions.tolist()):\n",
        "        print(f\"{tag.ljust(max_len_tag)}\\t{round(start_prob, 2)}\\t{round(end_prob, 2)}\")\n",
        "    print()\n",
        "    print(\"Between tags transitions:\")\n",
        "    persons_i = [i for i, tag in enumerate(TAG.vocab.itos) if \"PER\" in tag or tag == \"O\"]\n",
        "    max_len_tag = max([len(tag) for tag in TAG.vocab.itos if \"PER\" in tag ])\n",
        "    transitions = m.crf.transitions\n",
        "    to_tags = \"TO\".rjust(max_len_tag) + \"\\t\" + \"\\t\".join([tag.ljust(max_len_tag) for tag in tags if \"PER\" in tag or tag == \"O\"])\n",
        "    print(to_tags)\n",
        "    print(\"FROM\")\n",
        "    for from_tag_i, from_tag_probs in enumerate(transitions[persons_i]):\n",
        "        to_tag_str = f\"{tags[persons_i[from_tag_i]].ljust(max_len_tag)}\"\n",
        "        for to_tag_prob in from_tag_probs[persons_i]:\n",
        "            to_tag_str += f\"\\t{str(round(to_tag_prob.item(), 2)).ljust(max_len_tag)}\"\n",
        "        print(to_tag_str)\n",
        "\n",
        "print_crf_transitions(TEXT, model)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5lWbTw3yMiyr"
      },
      "source": [
        "# Training\n",
        "\n",
        "The outputs of the model are two lists: predictions and losses.\n",
        "These changes are taken into account by modifying the function that computes the accuracy.\n",
        "\n",
        " - Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "OVd5QFbXfLJf"
      },
      "outputs": [],
      "source": [
        "def opt(model, lr=1e-5, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0):\n",
        "    param_optimizer = list(model.named_parameters())\n",
        "    no_decay = ['bias', 'gamma', 'beta']\n",
        "    optimizer_grouped_parameters = [\n",
        "        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n",
        "            'weight_decay_rate': weight_decay_rate},\n",
        "        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n",
        "            'weight_decay_rate': second_weight_decay_rate}]\n",
        "    return optim.Adam(\n",
        "        optimizer_grouped_parameters,\n",
        "        lr=lr,\n",
        "        eps=eps\n",
        "    )\n",
        "\n",
        "optimizer = opt(model, lr=1e-5, eps=1e-6, weight_decay_rate=0.001, second_weight_decay_rate=0.0)\n",
        "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = TAG_PAD_IDX)\n",
        "model = model.to(device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ar0BMGWqfLJg"
      },
      "source": [
        " - Metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "JKJETj56fLJg"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score, classification_report\n",
        "\n",
        "def f1_loss( preds, y, tag_pad_idx, full_report=False):\n",
        "    index_o = TAG.vocab.stoi[\"O\"]\n",
        "    positive_labels = [i for i in range(len(TAG.vocab.itos))\n",
        "                       if i not in (tag_pad_idx, index_o)]\n",
        "\n",
        "    flatten_preds = [pred for sent_pred in preds for pred in sent_pred]\n",
        "\n",
        "    positive_preds = [pred for pred in flatten_preds\n",
        "                      if pred not in (tag_pad_idx, index_o)]\n",
        "\n",
        "    flatten_y = [tag for sent_tag in y for tag in sent_tag]\n",
        "    if full_report:\n",
        "      \n",
        "        positive_names = [TAG.vocab.itos[i]\n",
        "                              for i in range(len(TAG.vocab.itos))\n",
        "                              if i not in (tag_pad_idx, index_o)]\n",
        "        print(classification_report(\n",
        "                y_true=flatten_y,\n",
        "                y_pred=flatten_preds,\n",
        "                labels=positive_labels,\n",
        "                target_names=positive_names\n",
        "            ))\n",
        "\n",
        "    return f1_score(\n",
        "            y_true=flatten_y,\n",
        "            y_pred=flatten_preds,\n",
        "            labels=positive_labels,\n",
        "            average=\"micro\"\n",
        "        ), flatten_preds, flatten_y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "Wsulxo3ffLJg"
      },
      "outputs": [],
      "source": [
        "def train(model, iterator, optimizer, tag_pad_idx):\n",
        "    epoch_loss = 0\n",
        "    epoch_f1 = 0    \n",
        "    model.train()\n",
        "    \n",
        "    for batch in iterator:\n",
        "        \n",
        "        text = batch.text\n",
        "        tags = batch.tag\n",
        "      \n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        pred_tags_list, batch_loss = model(text, tags)\n",
        "        \n",
        "        # to calculate loss and F1 score we flatten true tags\n",
        "        true_tags_list = [\n",
        "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
        "                for sent_tag in tags.permute(1, 0).tolist()\n",
        "            ]\n",
        "        f1,_,_ = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx)\n",
        "\n",
        "        batch_loss.backward()\n",
        "        \n",
        "        optimizer.step()\n",
        "        epoch_loss += batch_loss.item()\n",
        "        epoch_f1 += f1\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_f1 / len(iterator)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def evaluate(model, iterator, tag_pad_idx,full_report):\n",
        "    epoch_loss = 0\n",
        "    epoch_f1 = 0\n",
        "    \n",
        "    model.eval()\n",
        "    preds = []\n",
        "    labels = []\n",
        "    with torch.no_grad():\n",
        "    \n",
        "        for batch in iterator:\n",
        "\n",
        "            text = batch.text\n",
        "            tags = batch.tag\n",
        "        \n",
        "            pred_tags_list, batch_loss = model(text, tags)\n",
        "            true_tags_list = [\n",
        "                [tag for tag in sent_tag if tag != TAG_PAD_IDX]\n",
        "                for sent_tag in tags.permute(1, 0).tolist()\n",
        "                ]\n",
        "            \n",
        "            f1, pred, lab = f1_loss(pred_tags_list, true_tags_list, tag_pad_idx, full_report)\n",
        "            preds.append(pred)\n",
        "            labels.append(lab)\n",
        "            epoch_loss += batch_loss.item()\n",
        "            epoch_f1 += f1\n",
        "        \n",
        "    return epoch_loss / len(iterator), epoch_f1 / len(iterator),preds, labels"
      ],
      "metadata": {
        "id": "jIiwJUceriK8"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs\n",
        "\n",
        "N_EPOCHS = 1"
      ],
      "metadata": {
        "id": "1IzhQMIO4VYM"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Tpj8lBEhfLJg",
        "outputId": "0549889e-e0e2-4083-9957-d32da4090ff3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 01 | Epoch Time: 1m 19s\n",
            "\tTrain Loss: 164.082 | Train F1 score: 14.66%\n",
            "\t Val. Loss: 134.863 |  Val. F1 score: 16.52%\n"
          ]
        }
      ],
      "source": [
        "t_loss = []\n",
        "t_f1 = []\n",
        "v_loss = []\n",
        "v_f1 = []\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "for epoch in range(N_EPOCHS):\n",
        "    start_time = time.time()\n",
        "   \n",
        "    train_loss, train_f1 = train(model, train_iterator, optimizer, TAG_PAD_IDX)\n",
        "    t_loss.append(train_loss)\n",
        "    t_f1.append(train_f1) \n",
        "    \n",
        "    valid_loss, valid_f1,_,_ = evaluate(model, valid_iterator, TAG_PAD_IDX, full_report= False)\n",
        "    v_loss.append(valid_loss)\n",
        "    v_f1.append(valid_f1)\n",
        "    \n",
        "    scheduler.step()\n",
        "    end_time = time.time()\n",
        "\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'tut3-model_conll.pt')\n",
        "    \n",
        "\n",
        "    if epoch%1 == 0: \n",
        "        print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
        "        print(f'\\tTrain Loss: {train_loss:.3f} | Train F1 score: {train_f1*100:.2f}%')\n",
        "        print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1 score: {valid_f1*100:.2f}%')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "id": "B3Jez-hKfLJh",
        "outputId": "4081dfeb-3a4e-42ca-d0fc-636b982d4657"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f89852fd1c0>"
            ]
          },
          "metadata": {},
          "execution_count": 24
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAELCAYAAAA4HCbKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAAd7ElEQVR4nO3df1SUZd4/8PcM41AKOEJIAz5qWvBQmhqz2Rpoh9G0+GF5dG3xx4YWlcZiLj61VlAI2qidTVNjbZ/KjpSVJUaQYG2bZWqaYmvW+giYCCgwiPJDGJi5vn/4bU4E4swwDDNe79c5nuPc1zX3/flQvufmmnvuUQghBIiI6Jqm7OsCiIio9zHsiYgkwLAnIpIAw56ISAIMeyIiCTDsiYgkwLAnIpIAw56kFx0djW+++aavyyDqVQx7IiIJMOyJumAymZCVlYXIyEhERkYiKysLJpMJAFBXV4fHHnsMOp0Od955JxISEmCxWAAAmzdvRlRUFMaNG4epU6di3759fdkGkZWqrwsgckevvfYajh49ip07d0KhUGDRokXYtGkTlixZgjfffBNBQUHWID969CgUCgVKS0uRk5OD7du3IygoCGfOnLG+CBD1NZ7ZE3UhLy8PixcvRkBAAPz9/bF48WJ8/PHHAACVSoWamhpUVlaiX79+0Ol0UCgU8PLygslkQklJCdra2jBkyBAMHTq0jzshuoxhT9SF6upqBAcHWx8HBwejuroaALBw4UIMGzYMCxYsgF6vx+bNmwEAw4YNw/Lly/Hqq69iwoQJeOqpp3Du3Lk+qZ/otxj2RF0YPHgwKisrrY+rqqowePBgAICPjw+eeeYZfP7553jttdfw5ptvWpd04uLi8O677+KLL76AQqHA2rVr+6R+ot9i2BMBaGtrQ2trq/VPTEwMXnvtNdTV1aGurg4bN25EXFwcAOCLL77Azz//DCEEfH194eXlZV2z37dvH0wmE9RqNby9vaFU8p8YuQe+QUsEICkpqcPjBQsWYNSoUYiPjwcATJs2DYsWLQIA/Pzzz1ixYgXq6urg5+eHP/7xj7jrrrvw008/4eWXX0ZJSQn69euHcePGISMjw+W9EHVFwS8vISK69vF3TCIiCTDsiYgkwLAnIpIAw56ISAIMeyIiCdh06aXBYEBhYSEqKiqQl5eH0NBQAEBraytWrlyJffv2wdvbG2PHjsWKFSsAXL5t7C/XGgNAamoqoqKieqkNIiLqjk1hr9frMX/+fMyZM6fD9jVr1sDb2xuFhYVQKBSora3tML5+/XrrC4Mjzp9vgsXiOVeGBgT4wGhs7OsyXIo9y4E9ewalUoFBgwZ0OWZT2Ot0uk7bmpqakJubiy+//BIKhQIAcMMNN/SgzM4sFuFRYQ/A4+p1BvYsB/bs2Rz+BG15eTk0Gg02bNiAAwcOYMCAAUhJSenwwpCamgohBCIiIrB06VL4+fk5pWgiIrKPw2FvNptRXl6OW2+9FU8//TSOHj2Kxx9/HLt374aPjw9ycnKg1WqtXwKRkZFh902hAgJ8HC2vzwQG+vZ1CS7HnuXAnj2bw2Gv1WqhUqkQGxsLABgzZgwGDRqEsrIyjB49GlqtFgCgVquRkJCAJ554wu5jGI2NHvVrVGCgL2pqGvq6DJdiz3KwtWchBM6fr4HJ1ALAc/7tdkWpVLrpl88ooFZfh0GDAq1L6L9QKhVXPEl2OOz9/f0xfvx47N27F5GRkSgrK4PRaMSwYcPQ3NwMs9kMX19fCCFQUFCA8PBwRw9FRB6isfECFAoFgoKGQKHw7Cu7VSol2tvdL+yFsKC+vhaNjRfg66ux+Xk23QgtMzMTRUVFqK2txaBBg6DRaJCfn4/y8nIsX74c9fX1UKlUWLJkCSZNmoTy8nIkJyfDbDbDYrFg5MiReO6556z3A7cVz+zdH3uWg609V1efgb9/EFSqfi6oqne5a9gDQHt7G+rqzmHw4CEdtnd3Zu/Wd71k2Ls/9iwHW3s+e/ZnBAUN7bS84IncOeyFEDh37jRuvHFYh+3dhb1n/55FRG7nWgh6d+fIz5hfXkJE16RHH/0T2tra0N7ehvLy07jpppEAgNDQMCxfnm7TPnJzt6O1tRWzZ8+5+uRfmTkzDqtX/w0jRtxsd929hWFPRNek11/fAgCoqqrEI4/Mw1tvvdNpTnt7O1SqK8fgAw/M7LX6XI1hT0RSmTkzDnr9vTh8+CBGjLgZSUmL8MILz6KpqQkmkwkTJtyNRYtSAAD/+79/x6VLl/Dkk0tQUJCH3bt3wdfXD6WlJfD19UFm5moEBHR/54AzZ8qxZs1K1Nefh5eXF5KSFuOuuyagpaUFmZnpOHWqFF5eKgwdOgwrVryE06dPISvrRbS0tMBiMeO+++KQkDCvx30z7ImoV+z9dxW+/r6qV/YdebsWd4/WOvz8pqYmvP762wAu39DRYPgb+vfvj/b2dixd+iT27/8GkZGRnZ7344/HsWXLuwgKuhEGQya2b38Pjz22uNtjvfjic5g+/UHExj6AsrJSPPnko9i6dTu+/74Yzc1N2Lr1AwDAxYsXAQAffbQdkZETMW9eYoftPcWwJyLpTJsWY/27xWLBpk3r8O9/fw9AwGg04v/+70SXYX/77WMQFHQjAOC220bh4MED3R6nubkJJ0+ewP33X/7i+ptuGoGbbw7DDz/8GzfffAtOnSrDyy8bMG5cBCZMuHy8sWPHYdOm9WhpacEdd+hwxx2d703mCIY9EfWKu0f37Oy7N/Xvf7317++9l4OGhovYvPkteHt7w2DIgsnU2uXz1Gq19e9KpRfMZrPDNYSEDMHWre/j0KGD2L9/LzZv3ogtW7bhnnv0GDXqdnz77X5s3foW8vM/RlraCoePY623x3sgIvJgDQ0NCAi4Ad7e3qipqcbXX3/ptH337z8AN98cik8//QQAcOpUGUpKTuC220ajuvoclEovTJx4D/7857+gvv48Ghou4syZcvj7B+D+++OQmPgojh//wSm18MyeiKQ2a9ZDeP75pzFv3h8QGBiEiIjfOXX/6emZWLNmJd5//x14eXnhuecyMGjQIOzbtxfZ2RsAABaLGXPnPowbbgjE22+/gaKiXejXTwWFQoGUlL84pQ5+gtaJ+MlKObDnKzt79udOn+r0VO78CVqg6581P0FLRCQ5hj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYE9E16S9/+TNyc7d32CaEwKxZ03HkyHdXfF5W1gv48MP3AFy+n/177+V0Oa+gIA/PPfc/XY7NnBmH0tKTDlbeOxj2RHRNiomJR0HBJx22HTnyHZRKBcaOvcOmfTzwwEy7v7jEXfF2CUTUK9pO7EXbf/b0yr77hU1Ev9C7u50TFTUJL7+8CqdOlWH48JsAAPn5H+P+++NQWlqCl19+CS0tl2AymRAf/yD+8IeETvv49f3s29ra8Le/rcbhw4cwcKAGt9wSZlOtvJ89EVEv6tevH6ZMuQ8FBR9j0aIUNDc34auvvsTWre/Dx8cHr7yyCWq1Gs3NzUhK+hPuvPP31heFruzc+SGqqiqxdesHaG9vx+LFj0KrvfpdPXk/eyK6pvULvfuqZ9+9LSYmHqmpyXjssSfx+ee7MXr0GAweHIS6OiM2bHgJJ0+egEKhRG1tDU6ePNFt2B8+/B3uuy8WKpUKKpUKU6feh++/L+72+O50P3uu2RPRNeuWW0IREBCI/fu/QUHBx4iJuRy6f//7Rvj7B+CNN3KwZcu7CA+/DSaTyaW1/XI/+9/9bjwOHTqAhx/+I1pbW3HPPXps2vSP/z/+FlasSHPK8Rj2RHRNi4mJxxtvbEZ5+WlERU0CADQ2NmDw4CCoVCqUlp7E0aPFV91PRIQOu3YVoL29Ha2tLdi9e9dVn8P72RMRuciUKdOwceM6xMc/iH79+gEA/vSnhVixIg35+TvxX/81FGPHjrvqfuLjZ+DkyZOYO3cWBg7U4L//+zacP2+86vM85n72BoMBhYWFqKioQF5eHkJDQwFc/pLelStXYt++ffD29sbYsWOxYsXlr84qKyvDM888g/r6emg0GhgMBgwfPtzu4ng/e/fHnuXA+9m7H3vvZ3/VM3u9Xo/58+djzpyO15quWbMG3t7eKCwshEKhQG1trXUsPT0dCQkJmD59Onbu3Im0tDS8/fbbjvRDREROcNU1e51O1+nyoqamJuTm5iIlJQUKhQIAcMMNNwAAjEYjjh8/jtjYWABAbGwsjh8/jrq6OmfXTkRENnJozb68vBwajQYbNmzAgQMHMGDAAKSkpECn06GqqgpBQUHw8vICAHh5eWHw4MGoqqqCv7+/Xce50q8j7iww0LevS3A59iwHW3qurlbCy0thPQn0dCqVe17DIoSAUqm06/9Dh8LebDajvLwct956K55++mkcPXoUjz/+OHbv3u3I7q6Ia/bujz3LwfaeFWhtNUGl6tfrNfU2d16zb29vA6Do9N/E6d9Bq9VqoVKprEs1Y8aMwaBBg1BWVgatVotz587BbDYDuPzCUF1dbdMnzYjIs11/vQ8aGuohhHuG5LVACAsaGs7j+uvtW/lw6Mze398f48ePx969exEZGYmysjIYjUYMGzYMfn5+CA8PxyeffILp06fjk08+QXh4uN1LOETkeXx8BuL8+RqcO3cGgOf8Vt4VpVIJi8UdX7QUUKuvg4/PQPuedbVLLzMzM1FUVITa2loMGjQIGo0G+fn5KC8vx/Lly1FfXw+VSoUlS5Zg0qTLH1goKSnBM888g4sXL8LPzw8GgwEjRoywuyUu47g/9iwH9uwZulvGuWrY9yWGvftjz3Jgz57B6Wv2RETkWRj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBFS2TDIYDCgsLERFRQXy8vIQGhoKAIiOjoZarYa3tzcAIDU1FVFRUQCAsLAwhIaGQqm8/HqyevVqhIWF9UYPRER0FTaFvV6vx/z58zFnzpxOY+vXr7eG/29t27YNAwYM6FmFRETUYzaFvU6n6+06iIioF9kU9t1JTU2FEAIRERFYunQp/Pz8rGPz5s2D2WzGxIkTkZycDLVabde+AwJ8elqeywUG+vZ1CS7HnuXAnj2bQgghbJ0cHR2N7Oxs67JNVVUVtFotTCYTsrKy0NTUhLVr13YYa2xsxLJlyxAaGoqnnnrKruKMxkZYLDaX1+cCA31RU9PQ12W4FHuWA3v2DEql4oonyT26Gker1QIA1Go1EhIScPjw4U5jPj4+mDVrVocxIiJyLYfDvrm5GQ0Nl1/1hBAoKChAeHg4AODChQtoaWkBALS3t6OwsNA6RkRErmfTmn1mZiaKiopQW1uLxMREaDQaZGdnIzk5GWazGRaLBSNHjkR6ejoAoLS0FGlpaVAoFGhvb8e4ceOQkpLSq40QEdGV2bVm72pcs3d/7FkO7Nkz9NqaPREReQaGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUlAZcskg8GAwsJCVFRUIC8vD6GhoQCA6OhoqNVqeHt7AwBSU1MRFRUFACguLkZaWhpaW1sREhKCNWvWICAgoJfaICKi7th0Zq/X65GTk4OQkJBOY+vXr8fOnTuxc+dOa9BbLBYsW7YMaWlpKCwshE6nw9q1a51bORER2cymsNfpdNBqtTbv9NixY/D29oZOpwMAPPTQQ9i1a5djFRIRUY/ZtIzTndTUVAghEBERgaVLl8LPzw9VVVUIDg62zvH394fFYkF9fT00Gk1PD0lERHbqUdjn5ORAq9XCZDIhKysLGRkZTl2uCQjwcdq+XCUw0LevS3A59iwH9uzZehT2vyztqNVqJCQk4IknnrBur6ystM6rq6uDUqm0+6zeaGyExSJ6UqJLBQb6oqamoa/LcCn2LAf27BmUSsUVT5IdvvSyubkZDQ2XfxBCCBQUFCA8PBwAMGrUKLS0tODQoUMAgG3btmHatGmOHoqIiHrIpjP7zMxMFBUVoba2FomJidBoNMjOzkZycjLMZjMsFgtGjhyJ9PR0AIBSqcTq1auRnp7e4dJLIiLqGwohhNuuk3AZx/2xZzmwZ8/QK8s4RETkORj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBBj2REQSYNgTEUmAYU9EJAGGPRGRBGwKe4PBgOjoaISFheHEiROdxjds2NBpLCwsDHFxcZg+fTqmT5+O//znP86rmoiI7KKyZZJer8f8+fMxZ86cTmM//PADiouLERIS0mls27ZtGDBgQM+rJCKiHrHpzF6n00Gr1XbabjKZkJGRgRdeeMHZdRERkRPZdGZ/JevWrUN8fDyGDBnS5fi8efNgNpsxceJEJCcnQ61W9+RwRETkIIfD/siRIzh27BhSU1O7HP/Xv/4FrVaLxsZGLFu2DBs3bsRTTz1l1zECAnwcLa/PBAb69nUJLsee5cCePZvDYX/w4EGUlJRAr9cDAM6ePYuFCxdi1apViIyMtC77+Pj4YNasWXjzzTftPobR2AiLRThaossFBvqipqahr8twKfYsB/bsGZRKxRVPkh0O+6SkJCQlJVkfR0dHIzs7G6Ghobhw4QK8vb1x3XXXob29HYWFhQgPD3f0UERE1EM2hX1mZiaKiopQW1uLxMREaDQa5OfnX3F+aWkp0tLSoFAo0N7ejnHjxiElJcVpRRMRkX0UQgi3XSfhMo77Y89yYM+eobtlHH6ClohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAjaFvcFgQHR0NMLCwnDixIlO4xs2bOg0VlxcjPj4eEydOhULFiyA0Wh0XtVERGQXm8Jer9cjJycHISEhncZ++OEHFBcXdxizWCxYtmwZ0tLSUFhYCJ1Oh7Vr1zqvaiIisotNYa/T6aDVajttN5lMyMjIwAsvvNBh+7Fjx+Dt7Q2dTgcAeOihh7Br166eV0tERA7p0Zr9unXrEB8fjyFDhnTYXlVVheDgYOtjf39/WCwW1NfX9+RwRETkIJWjTzxy5AiOHTuG1NRUZ9bTQUCAT6/tu7cEBvr2dQkux57lwJ49m8Nhf/DgQZSUlECv1wMAzp49i4ULF2LVqlXQarWorKy0zq2rq4NSqYRGo7HrGEZjIywW4WiJLhcY6Iuamoa+LsOl2LMc2LNnUCoVVzxJdjjsk5KSkJSUZH0cHR2N7OxshIaGwmKxoKWlBYcOHYJOp8O2bdswbdo0Rw9FREQ9ZFPYZ2ZmoqioCLW1tUhMTIRGo0F+fv4V5yuVSqxevRrp6elobW1FSEgI1qxZ47SiiYjIPgohhNuuk3AZx/2xZzmwZ8/Q3TIOP0FLRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBhj0RkQQY9kREEmDYExFJgGFPRCQBlS2TDAYDCgsLUVFRgby8PISGhgIAFi1ahDNnzkCpVKJ///54/vnnER4eDgCIjo6GWq2Gt7c3ACA1NRVRUVG91AYREXXHprDX6/WYP38+5syZ02G7wWCAr68vAOCzzz7D8uXLsWPHDuv4+vXrrS8MRETUd2wKe51O1+X2X4IeABobG6FQKJxTFREROZVNYd+dZ599Fnv37oUQAv/4xz86jKWmpkIIgYiICCxduhR+fn527TsgwKen5blcYKDv1SddY9izHNizZ1MIIYStk6Ojo5Gdnd3l0kxubi7y8/Px+uuvAwCqqqqg1WphMpmQlZWFpqYmrF271q7ijMZGWCw2l9fnAgN9UVPT0NdluBR7lgN79gxKpeKKJ8lOuxrngQcewIEDB3D+/HkAgFarBQCo1WokJCTg8OHDzjoUERHZyeGwb2pqQlVVlfXxP//5TwwcOBAajQbNzc1oaLj8iiiEQEFBgfUqHSIicj2b1uwzMzNRVFSE2tpaJCYmQqPRYMuWLUhJScGlS5egVCoxcOBAZGdnQ6FQwGg0Ijk5GWazGRaLBSNHjkR6enpv90JERFdg15q9q3HN3v2xZzmwZ8/gkjV7IiJyXwx7IiIJMOyJiCTAsCcikkCPP0Hbm5RKz7v9gifW3FPsWQ7s2f11V69bX41DRETOwWUcIiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOyJiCTAsCcikgDDnohIAgx7IiIJMOztdOnSJSxZsgRTpkzBtGnT8MUXX1xx7vvvv48pU6Zg8uTJyMjIgMVi6TDe2tqKmJgYzJgxo7fL7hFn9PzZZ59hxowZiI2NRUxMDN544w1XlW+zsrIyzJ49G1OnTsXs2bNx6tSpTnPMZjNefPFFTJ48GVOmTMEHH3xg05i76mnPGzduRExMDOLi4jBjxgx89dVXLqzefj3t9xelpaUYM2YMDAaDC6p2EkF2efXVV8Wzzz4rhBCirKxMTJgwQTQ2Nnaad/r0aREVFSWMRqMwm81iwYIFYseOHR3mrFq1Svz1r38VDz74oCtKd5gzei4uLhZnz54VQghx8eJFMXnyZHHw4EGX9WCLefPmidzcXCGEELm5uWLevHmd5uzYsUMsWLBAmM1mYTQaRVRUlCgvL7/qmLvqac979uwRzc3NQgghfvzxRxERESEuXbrkugbs1NN+hRCivb1dzJ07VyxdulS89NJLLqu9p3hmb6dPP/0Us2fPBgAMHz4co0aNwp49ezrNKywsxOTJk+Hv7w+lUolZs2ahoKDAOn7o0CGcOnUK06dPd1ntjnJGz2PGjEFQUBAAwNfXFyNHjkRFRYXrmrgKo9GI48ePIzY2FgAQGxuL48ePo66ursO8goICzJo1C0qlEv7+/pg8eTJ27dp11TF35Iyeo6KicP311wMAwsLCIIRAfX29S/uwlTP6BYDNmzfjnnvuwfDhw11Zfo8x7O1UWVmJkJAQ62OtVouzZ892mldVVYXg4GDr4+DgYOsXtDc3N2PlypV48cUXe79gJ3BGz79WUlKC4uJi3HXXXb1TsAOqqqoQFBQELy8vAICXlxcGDx7cqf7f9vjrn0V3Y+7IGT3/Wm5uLoYOHYobb7yxdwt3kDP6/emnn/D111/j4YcfdlndzuLWtzjuCw8++CAqKyu7HPvmm2+ccozVq1cjISEBQUFBXa4Zuporev5FdXU1Fi1ahPT0dOuZPnm+b7/9FuvWrXPL92Kcpa2tDc8//zxWrVplfcHwJAz739ixY0e348HBwaioqIC/vz+Ay2cB48eP7zRPq9V2CNDKykpotVoAwHfffYc9e/Zg06ZNaG1txYULFxAXF4e8vDwndmI7V/QMXP41OjExEY888gjuu+8+J1XvHFqtFufOnYPZbIaXlxfMZjOqq6s71P/LvMrKStx+++0AOp4FdjfmjpzRMwAcOXIEy5Ytw6ZNmzBixAiX9mCPnvZbU1OD06dPIykpCQBw8eJFCCHQ2NiIFStWuLwfu/X1mwaeZv369R3erPz9738vGhoaOs3r6s3Kjz76qNO8/fv3u/0btM7oua6uTsTFxYmcnByX1m6PuXPndnjzbu7cuZ3mfPjhh53evDt9+vRVx9xVT3s+evSomDRpkiguLnZp3Y7qab+/tn79eo96g5Zhb6empiaRnJwsJk+eLO69916xe/du69grr7wi3nnnHevjd999V+j1eqHX60VaWppob2/vtD9PCHtn9PzSSy+J0aNHi/j4eOuf7du3u7yX7pw8eVLMnDlT3HvvvWLmzJmipKRECCHEI488Ir7//nshxOUrMdLS0qw9btu2zfr87sbcVU97njFjhhg/fnyH/64//fRTn/Rii572+2ueFvb8pioiIgnwahwiIgkw7ImIJMCwJyKSAMOeiEgCDHsiIgkw7ImIJMCwJyKSAMOeiEgC/w8e52t3PiYOLgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt \n",
        "import numpy as np \n",
        "sns.set()\n",
        "\n",
        "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
        "\n",
        "plt.plot(x,t_loss)\n",
        "plt.plot(x,v_loss)\n",
        "plt.title(\"Loss\")\n",
        "plt.legend([\"Train loss\", \"Valid loss\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "id": "voPrr4eefLJh",
        "outputId": "e58eef34-3df3-4419-8a55-88d4f970b030"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f8988068af0>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAELCAYAAAAlTtoUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAAsTAAALEwEAmpwYAAApuElEQVR4nO3de3RU1f3//2dmklA1QUmcDBOM3H4tzJKgIsUiBGxJJMKEiVE+4culpQgUUfMFvCRqJVIbIfQnVVBodSnIt94qxUSTmGq6XEKo1KaioAHFQIpNJhcSkaRQQmbO9w8W8zUNkBNyI8zrsZZrTc7e5+z9nkhec/aZmRNkGIaBiIiICZaenoCIiPQeCg0RETFNoSEiIqYpNERExDSFhoiImKbQEBER0xQaIiJiWnBPT0DkQvSTn/yEw4cPY7Va/dsKCwux2+08+uijfPTRR/zzn//kiSeeICUlpQdnKtK9FBoiZ/G73/2Om266qdX24cOHM2XKFH7zm9/0wKxaam5uJjhY/4yl+2h5SqSdZs2axdixY+nTp0+bfT/44AOmTJnC9ddfT1xcHC+88IK/raioCLfbzahRo4iPj2fbtm0AVFdXs2jRIsaMGUNCQgJ//OMf/fusW7eOtLQ07r//fkaNGsWbb75JQ0MDDz/8MOPHjycuLo7f/va3eL3ezi9cBJ1piHSpRx55hKeeeorRo0fz7bff8q9//QuA3bt3k56eztq1axk7diy1tbU0NjYCsGzZMr7//e+zfft2Dhw4wM9//nNiYmIYO3YsAH/5y194+umnWb16NU1NTdx3331ERkby7rvvcvz4cX7xi1/gcDiYMWNGj9UtFy+daYicxd13383o0aMZPXo0ixcvPq9jBAcH89VXX9HY2Mjll1/ONddcA8CWLVu4/fbbGTduHBaLBbvdztChQ/F4PHz88cfcf//99OnTB6fTyfTp08nNzfUf87rrriM+Ph6LxUJjYyMffPABDz/8MJdeeimRkZHMnTuX/Pz8TnkORP6bzjREzuLZZ5894zWN9li7di0bNmzgySefZNiwYdx3331cf/31eDweJk6c2Kp/TU0Nl19+OWFhYf5t0dHRfPbZZ/6f+/fv739cWVlJc3Mz48eP92/z+Xw4HI4OzVvkbBQaIl1o5MiRbNiwgZMnT/Lyyy+zZMkSPvjgAxwOB4cOHWrVPyoqim+//ZbGxkZ/cHg8Hux2u79PUFCQ/3H//v0JDQ1l586duiAu3ULLUyLt1NTUxIkTJzAMg+bmZk6cOIHP5ztjv7feeouGhgZCQkK47LLLsFhO/ZO744472Lp1Kx9++CE+n4/q6mrKyspwOBxcf/31rFmzhhMnTrBv3z62bNnCtGnTzjiXqKgoxo0bx6pVq2hsbMTn83Ho0CE++uijLn0OJHApNETa6c4772TkyJHs2rWLRx99lJEjR/L3v//9jH1zc3P5yU9+wqhRo3jttdf8b9MdOXIkK1eu5IknnuCGG25g9uzZVFZWArBmzRoqKiqIi4vjnnvu4d577z3nMtnq1as5efIkU6ZM4Yc//CFpaWnU1tZ2fuEiQJBuwiQiImbpTENERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYdtF/hPSbb/6Nz9e73lUcGRlGXV1jT0+jWwVazYFWL6jm3sJiCaJfv8vO2n7Rh4bPZ/S60AB65Zw7KtBqDrR6QTVfDLQ8JSIipik0RETEtIt+eUpELi6GYfDNN7U0Nf0HuLCXfmpqLGf8MsueF0Ro6Pfo18/W4luTzVBoiEiv0tj4LUFBQdjtVxEUdGEvlgQHW2huvvBCwzB8HDlymMbGbwkPv6Jd+17Yz7iIyH85fryR8PArLvjAuJAFBVkID+/H8ePtf2eXqWf94MGDpKamMnnyZFJTUykvL2/Vp7i4mJSUFEaMGEF2dnar9oKCApKSknC5XCQlJXH48GEA1q1bx9ixY3G73bjdblasWOHf5/jx4yxZsoSEhAQSExN5//33212giFxcfD4vVqsWSTrKag3G5/O2ez9Tz3xmZiYzZ87E7XaTm5vL8uXL2bx5c4s+MTExZGVlUVhYSFNTU4u2PXv28Mwzz/DSSy9hs9loaGggNDTU356cnEx6enqrcV944QXCwsJ47733KC8vZ9asWbz77rtcdtnZ30MsIhe/9q7DS2vn+xy2GRp1dXWUlpayceNGAFwuF48//jj19fVERET4+w0cOBCAoqKiVqGxadMm5s2bh81mAyA8PNzU5N555x1WrVoFwKBBgxgxYgTbtm3j1ltvNbW/iEhXWrDgZ5w8eZLm5pN8/fUhBg8eCsAPfjCMhx/ONHWMnJwtnDhxgtTUWe0a+557FlJdXe1/EX311QP51a9Wsn//F6xZk82XX37B2LHj+PWvV7evqDa0GRqn709stVoBsFqtREVF4fF4WoTGuZSVlXHVVVcxa9Ysjh07RkJCAnfddZc/6fLz8ykuLsZms3Hvvfdy/fXXA1BZWcmAAQP8x3E4HFRVVbW7SBGRrvD88y8B4PFUMn/+HDZteqVVn+bm5nPevz05+Y7zHn/JkvsZNy6uxbZ+/SK4555l7N//BSUlfzvvY59NtywMer1evvjiCzZu3EhTUxPz588nOjqa5ORkZsyYwaJFiwgJCWHHjh0sXryYgoIC+vXr1yljR0aGdcpxupvNZu5s7GISaDUHWr3QOTXX1FgIDr6wLoJbrRYgyD+v5OSpJCRM5h//+DtDh/5/LFp0N48++jD//ncjTU1N3HTTeO69dwkAzz//O44fP05a2lLy8t7i3XcLCQ8P58CBMsLCwlm16jdERl7ZasygoCCs1qBWz0X//nb697fz9dflBAW1bv8ui8XS7t9Jm6HhcDiorq7G6/VitVrxer3U1NTgcDhMDxIdHU1iYiKhoaGEhoYyadIkdu/eTXJysn/JCmDcuHE4HA7279/PmDFjiI6OpqKiwn9G4/F4uPHGG9tVYF1dY6/7GL/NFk5tbUNPT6NbBVrNgVYvdF7NPp+vxdtYd+zxULzb0+Hjnsn4kQ7Gxbb9t87r9QFGi3k1NDTy4ov/h+ZmHydOnGDVqjVceumlNDc3s2zZPRQXF/OjH93k/6qj5mYfPp9BaennvPTSq9jt/cnO/jWvvfYqv/jF3a3GNAyDJ5/8Db/73XoApk+fwdSp0/ztPp+BYRjnfMuvz+dr9TuxWILO+WK7zbiOjIzE6XSSl5cHQF5eHk6n0/TSFJy6DlJcXIxhGJw8eZKdO3cyfPhwAKqrq/399u7dS0VFBYMHDwYgMTGR119/HYDy8nL27NlDXFxc6wFERC4wiYlT/Y99Ph/r1z/Nz372v7jzztkcOFDG/v1fnnG/kSOvxW7vD8A114ygsvJfZx1jyZL72bTpFTZteqVFYHQlU8tTjz32GBkZGaxfv56+ffv631K7YMEC0tLSiI2NpaSkhGXLltHY2IhhGOTn55OVlUVcXBxTp07ls88+Y8qUKVgsFsaPH88dd5xax1uzZg2ff/45FouFkJAQVq9e7T/7uPPOO8nIyCAhIQGLxcKvfvUrwsJ653KTiHSNcbHmzga626WXXuJ//PrrL9PQcJTnnttEnz59yM7OoqnpxBn3++47Sy2WU6s7FxJToTF06FDeeOONVtuff/55/+PRo0ezbdu2M+5vsVh46KGHeOihh1q1nekzHaddeumlrF271swURUQuWA0NDURGXkmfPn2ora2huPgDkpNv7+lpnRd9QkZEpItNnz6DRx9NZ86c/8Fms3PDDT/ssrE8nkoWL57Pf/7zH5qaTnDbbVO4886FuFzJnXL8IMMwetdV4nbShfDeIdBqDrR6ofNqrqr6J/37D+yEGXW9C/W7p04703PZ4QvhIiIipyk0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRGR83TffWnk5Gxpsc0wDKZPd7Nr1z/Oul9W1mP86U+nvlcvJ2cLr7/+8hn7FRS8zS9/+eAZ2+65ZyHTp7uZO3cmc+fOZPnyU9+4sX//F9x11zwmTRp31n07Qp8IFxE5T1OnTuO11/7Q4p4Yu3b9A4sliOuuG2XqGLqfhohINzr55Q5OfnHm773rqJBhEwj5wbiztsfFTeTJJ1dSXn6QQYNOfTt3fv5bTJmSxIEDZaxZs4rjx4/T1NTEtGm38T//M7PVMV544fccP36ce+5ZwsmTJ/ntb1fz8cclXH75FXz/+8PaPecrr7Rx5ZU2/vnPg+3e1wyFhojIeQoJCSEh4VYKCt5i8eL/zbFj/2b79g/4wx/+SFhYGOvW/Q6LJZhjx46xcOHPGDNmrD9cziQ39094PJX84Q9v0NzczN13LzjnvYueeur/5/nnNwCt76fRVRQaItKrhfxg3DnPBrra1KnTuP/+e/nFL+7hL395j9jYa4mKslNfX8eTT65i//4vCQqycPhwLV999eU5Q+Pjj//Brbe6CA4OJjg4mMmTb2X37k/O2v9My1NdTRfCRUQ64Pvf/wGRkTZ27vwrBQVv+V/t//73zxIZeSUvvvgyL730Kk7nNTQ1NfXwbDtOoSEi0kFTp07jxRef4+uvDxEXNxGAxsYG7HY7wcHBHDjwFZ9++kmbx7nhhtEUFhbQ3NzMiRP/4b33Crt45u2n5SkRkQ5KSEjk2WefZtq02wgJCQHgZz+7k1//OpO33sohJuZqrrvu+jaPM21aCl999RWzZ0/n8suvYPjwa/jmm7p2zUX30+gg3U+jdwi0mgOtXtD9NC5EXXY/jYMHD5KamsrkyZNJTU2lvLy8VZ/i4mJSUlIYMWLEGW/hWlBQQFJSEi6Xi6SkJA4fPgzAs88+y9SpU0lKSiIlJYXt27f798nIyGDChAm43W7cbjcbNmwwM10REekippanMjMzmTlzJm63m9zcXJYvX87mzZtb9ImJiSErK4vCwsJWF3v27NnDM888w0svvYTNZqOhocF/8/SRI0cyb948LrnkEvbt28fs2bMpLi7me9/7HgALFy5k9uzZnVGriIh0UJtnGnV1dZSWluJyuQBwuVyUlpZSX1/fot/AgQNxOp0EB7fOoU2bNjFv3jxsNhsA4eHh9OnTB4C4uDguueQSAIYNG4ZhGBw5cqRDRYnIxe0iX1XvFuf7HLZ5puHxeLDb7VitVgCsVitRUVF4PB4iIiJMDVJWVsZVV13FrFmzOHbsGAkJCdx1110EBQW16JeTk8PVV19N//79/ds2btzI66+/TkxMDPfddx9Dhw5tT33nXJu7kNls4T09hW4XaDUHWr3QOTXX1YUQFOQjODikE2bU9YKDL8w3qTY3nyQ0NKTdv5NuefeU1+vliy++YOPGjTQ1NTF//nyio6NJTk729/noo494+umnefHFF/3bli5dis1mw2KxkJOTw/z58ykqKvIHmBm6EN47BFrNgVYvdF7NoaGX8s039VxxRSRBQRfmH+TTLtQL4Ybh48iROkJCLm31O2nrQniboeFwOKiursbr9WK1WvF6vdTU1Jzzo+3/LTo6msTEREJDQwkNDWXSpEns3r3bHxq7du3igQceYP369QwZMsS/n91u9z9OTk5m5cqVVFVVMWDAANNji8jFJSzscr75ppbq6n8BF/YLQovFgs934YUGBBEa+j3Cwi5v955thkZkZCROp5O8vDzcbjd5eXk4nU7TS1Nw6jrIBx98gNvtprm5mZ07dzJ58mQAdu/ezdKlS1m7di3XXHNNi/2qq6v9wbF9+3YsFkuLIBGRwBMUFERERFRPT8OUi/GM0tTnNMrKysjIyODo0aP07duX7OxshgwZwoIFC0hLSyM2NpaSkhKWLVtGY2MjhmEQHh5OVlYWcXFx+Hw+srOz2bZtGxaLhfHjx5Oeno7FYuH222+noqKiRRisXr2aYcOGMXfuXOrq6ggKCiIsLIwHH3yQ6667rl0Fanmqdwi0mgOtXlDNvUVby1P6cN8FqDf+j9ZRgVZzoNULqrm36JQP94mIiIBCQ0RE2kGhISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGmmQuPgwYOkpqYyefJkUlNTKS8vb9WnuLiYlJQURowYQXZ2dqv2goICkpKScLlcJCUlcfjwYQC8Xi8rVqwgPj6ehIQE3njjDf8+52oTEZHuF2ymU2ZmJjNnzsTtdpObm8vy5cvZvHlziz4xMTFkZWVRWFhIU1NTi7Y9e/bwzDPP8NJLL2Gz2WhoaCA0NBSAt99+m0OHDvHuu+9y5MgRkpOTGTt2LFddddU520REpPu1eaZRV1dHaWkpLpcLAJfLRWlpKfX19S36DRw4EKfTSXBw6xzatGkT8+bNw2azARAeHk6fPn2AU2cg06dPx2KxEBERQXx8PIWFhW22iYhI92szNDweD3a7HavVCoDVaiUqKgqPx2N6kLKyMr7++mtmzZrFbbfdxvr16zEMw3/86Ohof1+Hw0FVVVWbbSIi0v1MLU91lNfr5YsvvmDjxo00NTUxf/58oqOjSU5O7vKxIyPDunyMrmCzhff0FLpdoNUcaPWCar4YtBkaDoeD6upqvF4vVqsVr9dLTU0NDofD9CDR0dEkJiYSGhpKaGgokyZNYvfu3SQnJ+NwOKisrGTkyJFAy7OLc7WZVVfXiM9ntGufnmazhVNb29DT0+hWgVZzoNULqrm3sFiCzvliu83lqcjISJxOJ3l5eQDk5eXhdDqJiIgwPQmXy0VxcTGGYXDy5El27tzJ8OHDAUhMTOSNN97A5/NRX19PUVERkydPbrNNRES6n6nlqccee4yMjAzWr19P3759/W+pXbBgAWlpacTGxlJSUsKyZctobGzEMAzy8/PJysoiLi6OqVOn8tlnnzFlyhQsFgvjx4/njjvuAMDtdvPpp59yyy23AHD33XcTExPTZpuIiHS/IOP0FemLlJaneodAqznQ6gXV3Ft0eHlKRETkNIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYZuoe4QcPHiQjI4MjR45wxRVXkJ2dzaBBg1r0KS4uZs2aNXz55ZfMmTOH9PR0f9u6det45ZVXiIqKAmDUqFFkZmYCMHfuXL755hsAvF4v+/fvJzc3l+HDh5ORkcFf//pX+vXrB0BiYiJ33XVXh4sWEZHzYyo0MjMzmTlzJm63m9zcXJYvX87mzZtb9ImJiSErK4vCwkKamppaHSM5OblFkJy2adMm/+OioiKeeuophg8f7t+2cOFCZs+ebbYeERHpQm0uT9XV1VFaWorL5QLA5XJRWlpKfX19i34DBw7E6XQSHGwqh85oy5Yt3H777ee9v4iIdK02/8J7PB7sdjtWqxUAq9VKVFQUHo+HiIgI0wPl5+dTXFyMzWbj3nvv5frrr2/RXltby4cffsgTTzzRYvvGjRt5/fXXiYmJ4b777mPo0KGmxwSIjAxrV/8Lhc0W3tNT6HaBVnOg1Quq+WJw/qcF7TBjxgwWLVpESEgIO3bsYPHixRQUFPivVQDk5OQQFxfXIoiWLl2KzWbDYrGQk5PD/PnzKSoq8geYGXV1jfh8RqfW09VstnBqaxt6ehrdKtBqDrR6QTX3FhZL0DlfbLe5POVwOKiursbr9QKnLlbX1NTgcDhMT8JmsxESEgLAuHHjcDgc7N+/v0WfrVu3tlqastvtWCynppicnMyxY8eoqqoyPa6IiHSuNkMjMjISp9NJXl4eAHl5eTidznYtTVVXV/sf7927l4qKCgYPHuzf9vHHH9PQ0MCECRPOut/27duxWCzY7XbT44qISOcytTz12GOPkZGRwfr16+nbty/Z2dkALFiwgLS0NGJjYykpKWHZsmU0NjZiGAb5+flkZWURFxfHmjVr+Pzzz7FYLISEhLB69WpsNpv/+Fu3biU5ObnVslN6ejp1dXUEBQURFhbGhg0bOnShXUREOibIMIzeteDfTrqm0TsEWs2BVi+o5t6iw9c0RERETlNoiIiIaQoNERExTaEhIiKmKTRERMQ0hYaIiJim0BAREdMUGiIiYppCQ0RETFNoiIiIaQoNERExTaEhIiKmKTRERMQ0hYaIiJim0BAREdMUGiIiYppCQ0RETDMVGgcPHiQ1NZXJkyeTmppKeXl5qz7FxcWkpKQwYsQI/+1gT1u3bh1jx47F7XbjdrtZsWKFvy0jI4MJEyb42zZs2OBvO3z4MPPmzWPy5MlMmzaNTz/99DzLFBGRzmDqhtuZmZnMnDkTt9tNbm4uy5cvZ/PmzS36xMTEkJWVRWFhIU1NTa2OkZycTHp6+hmPv3DhQmbPnt1q+5NPPsno0aN58cUXKSkp4YEHHuDPf/4zQUFBZqYtIiKdrM0zjbq6OkpLS3G5XAC4XC5KS0upr69v0W/gwIE4nU6Cg03lkCmFhYXMmDEDgNGjRxMaGsqePXs67fgiItI+bYaGx+PBbrdjtVoBsFqtREVF4fF42jVQfn4+SUlJzJs3j127drVo27hxI0lJSSxevJiysjIAvvnmGwzDICIiwt/P4XBQVVXVrnFFRKTzdN5pwTnMmDGDRYsWERISwo4dO1i8eDEFBQX069ePpUuXYrPZsFgs5OTkMH/+fIqKijpt7MjIsE47Vney2cJ7egrdLtBqDrR6QTVfDNoMDYfDQXV1NV6vF6vVitfrpaamBofDYXoQm83mfzxu3DgcDgf79+9nzJgx2O12f1tycjIrV66kqqqKAQMGAFBfX+8/2/B4PPTv39/0uAB1dY34fEa79ulpNls4tbUNPT2NbhVoNQdavaCaewuLJeicL7bbXJ6KjIzE6XSSl5cHQF5eHk6ns8WyUVuqq6v9j/fu3UtFRQWDBw9u1bZ9+3YsFos/SBITE3nttdcAKCkp4T//+Q8jRowwPa6IiHSuIMMw2nwZXlZWRkZGBkePHqVv375kZ2czZMgQFixYQFpaGrGxsZSUlLBs2TIaGxsxDIPw8HCysrKIi4sjPT2dzz//HIvFQkhICGlpaUycOBGAuXPnUldXR1BQEGFhYTz44INcd911ANTW1vLAAw9QWVlJnz59WLFiBaNGjWpXgTrT6B0CreZAqxdUc2/R1pmGqdDozRQavUOg1Rxo9YJq7i06vDwlIiJymkJDRERMU2iIiIhpCg0RETFNoSEiIqYpNERExDSFhoiImKbQEBER0xQaIiJimkJDRERMU2iIiIhpCg0RETFNoSEiIqYpNERExDSFhoiImKbQEBER0xQaIiJimkJDRERMCzbT6eDBg2RkZHDkyBGuuOIKsrOzGTRoUIs+xcXFrFmzhi+//JI5c+aQnp7ub1u3bh2vvPIKUVFRAIwaNYrMzEwAVqxYwYcffkhoaCiXXnopjzzyCLGxsQDMmTOHyspKwsJO3Xrwpz/9KbfffnuHixYRkfNjKjQyMzOZOXMmbreb3Nxcli9fzubNm1v0iYmJISsri8LCQpqamlodIzk5uUWQnDZhwgQefvhhQkJCeP/991m6dClFRUX+9l/+8pf8+Mc/bm9dIiLSBdpcnqqrq6O0tBSXywWAy+WitLSU+vr6Fv0GDhyI0+kkONhUDvn9+Mc/JiQkBIDrrruOqqoqfD5fu44hIiLdo83Q8Hg82O12rFYrAFarlaioKDweT7sGys/PJykpiXnz5rFr164z9nn55Ze5+eabsVj+37RWr15NUlIS999/P9XV1e0aU0REOlf7TgvO04wZM1i0aBEhISHs2LGDxYsXU1BQQL9+/fx98vPzefvtt3n55Zf921avXo3D4cDr9fL73/+eJUuW8Oqrr7Zr7MjIsE6rozvZbOE9PYVuF2g1B1q9oJovBm2GhsPhoLq6Gq/Xi9Vqxev1UlNTg8PhMD2IzWbzPx43bhwOh4P9+/czZswYAN577z1++9vfsmnTJq688soWY8Ops5uf/vSnPPPMM/h8vhZnIm2pq2vE5zNM978Q2Gzh1NY29PQ0ulWg1Rxo9YJq7i0slqBzvthu869vZGQkTqeTvLw8APLy8nA6nURERJiexHeXlfbu3UtFRQWDBw8G4P3332flypW88MILXHXVVf5+zc3NHD582P9zfn4+P/jBD9oVGCIi0rmCDMNo82V4WVkZGRkZHD16lL59+5Kdnc2QIUNYsGABaWlpxMbGUlJSwrJly2hsbMQwDMLDw8nKyiIuLo709HQ+//xzLBYLISEhpKWlMXHiRAB+9KMfERIS0iKENm3aRJ8+fZg9ezYnT54EICoqikceeYQhQ4a0q0CdafQOgVZzoNULqrm3aOtMw1Ro9GYKjd4h0GoOtHpBNfcWHV6eEhEROU2hISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMc1UaBw8eJDU1FQmT55Mamoq5eXlrfoUFxeTkpLCiBEjyM7ObtG2bt06xo4di9vtxu12s2LFCn/b8ePHWbJkCQkJCSQmJvL++++bahMRke4XbKZTZmYmM2fOxO12k5uby/Lly9m8eXOLPjExMWRlZVFYWEhTU1OrYyQnJ5Oent5q+wsvvEBYWBjvvfce5eXlzJo1i3fffZfLLrvsnG0iItL92jzTqKuro7S0FJfLBYDL5aK0tJT6+voW/QYOHIjT6SQ42FQO+b3zzjukpqYCMGjQIEaMGMG2bdvabBMRke7XZmh4PB7sdjtWqxUAq9VKVFQUHo+nXQPl5+eTlJTEvHnz2LVrl397ZWUlAwYM8P/scDioqqpqs01ERLpf+04LztOMGTNYtGgRISEh7Nixg8WLF1NQUEC/fv26fOzIyLAuH6Mr2GzhPT2FbhdoNQdavaCaLwZthobD4aC6uhqv14vVasXr9VJTU4PD4TA9iM1m8z8eN24cDoeD/fv3M2bMGKKjo6moqCAiIgI4dWZz4403Apyzzay6ukZ8PqNd+/Q0my2c2tqGnp5Gtwq0mgOtXlDNvYXFEnTOF9ttLk9FRkbidDrJy8sDIC8vD6fT6f9DbkZ1dbX/8d69e6moqGDw4MEAJCYm8vrrrwNQXl7Onj17iIuLa7NNRES6X5BhGG2+DC8rKyMjI4OjR4/St29fsrOzGTJkCAsWLCAtLY3Y2FhKSkpYtmwZjY2NGIZBeHg4WVlZxMXFkZ6ezueff47FYiEkJIS0tDQmTpwIwLFjx8jIyGDv3r1YLBYeeOAB4uPj22wzS2cavUOg1Rxo9YJq7i3aOtMwFRq9mUKjdwi0mgOtXlDNvUWHl6dEREROU2iIiIhpCg0RETFNoSEiIqYpNERExDSFhoiImKbQEBER0xQaIiJimkJDRERMU2iIiIhpCg0RETFNoSEiIqYpNERExDSFhoiImKbQEBER0xQaIiJimkJDRERMU2iIiIhppkLj4MGDpKamMnnyZFJTUykvL2/Vp7i4mJSUFEaMGEF2dvYZj3PgwAGuvfbaFu1z587F7XbjdrtxuVwMGzaMffv2AZCRkcGECRP87Rs2bDiPEkVEpLMEm+mUmZnJzJkzcbvd5Obmsnz5cjZv3tyiT0xMDFlZWRQWFtLU1NTqGF6vl8zMTOLj41ts37Rpk/9xUVERTz31FMOHD/dvW7hwIbNnz25PTSIi0kXaPNOoq6ujtLQUl8sFgMvlorS0lPr6+hb9Bg4ciNPpJDj4zDn03HPPcfPNNzNo0KCzjrVlyxZuv/32dkxfRES6U5uh4fF4sNvtWK1WAKxWK1FRUXg8HtOD7Nu3j+LiYubOnXvWPrW1tXz44Ye43e4W2zdu3EhSUhKLFy+mrKzM9JgiItL5TC1PdcTJkyd59NFHWblypT94ziQnJ4e4uDgiIiL825YuXYrNZsNisZCTk8P8+fMpKio653H+W2RkWIfm31NstvCenkK3C7SaA61eUM0XgzZDw+FwUF1djdfrxWq14vV6qampweFwmBqgtraWQ4cOsXDhQgCOHj2KYRg0Njby+OOP+/tt3bqVBx98sMW+drvd/zg5OZmVK1dSVVXFgAEDTI0NUFfXiM9nmO5/IbDZwqmtbejpaXSrQKs50OoF1dxbWCxB53yx3WZoREZG4nQ6ycvLw+12k5eXh9PpbHFGcC7R0dH87W9/8/+8bt06jh07Rnp6un/bxx9/TENDAxMmTGixb3V1tT84tm/fjsViaREkIiLSvUwtTz322GNkZGSwfv16+vbt63/L7IIFC0hLSyM2NpaSkhKWLVtGY2MjhmGQn59PVlYWcXFxbR5/69atJCcnt1p2Sk9Pp66ujqCgIMLCwtiwYcNZL7SLiEjXCzIMo3et3bSTlqd6h0CrOdDqBdXcW7S1PKVPhIuIiGkKDRERMU2hISIipik0RETENIWGiIiYptAQERHTFBoiImKaQkNERExTaIiIiGkKDRERMU2hISIipik0RETENIWGiIiYdtF/z7jFEtTTUzgvvXXeHRFoNQdavaCae4O25nvRfzW6iIh0Hi1PiYiIaQoNERExTaEhIiKmKTRERMQ0hYaIiJim0BAREdMUGiIiYppCQ0RETFNoiIiIaQqNHnD8+HGWLFlCQkICiYmJvP/++2ft+8c//pGEhATi4+P51a9+hc/na9F+4sQJpk6dSkpKSldPu0M6o+aioiJSUlJwuVxMnTqVF198sbumb9rBgwdJTU1l8uTJpKamUl5e3qqP1+tlxYoVxMfHk5CQwBtvvGGq7ULV0ZqfffZZpk6dSlJSEikpKWzfvr0bZ39+OlrzaQcOHODaa68lOzu7G2bdSQzpduvWrTMeeeQRwzAM4+DBg8ZNN91kNDY2tup36NAhIy4uzqirqzO8Xq8xb948480332zRZ+XKlcZDDz1k3Hbbbd0x9fPWGTV/8sknRlVVlWEYhnH06FEjPj7e+Pvf/95tNZgxZ84cIycnxzAMw8jJyTHmzJnTqs+bb75pzJs3z/B6vUZdXZ0RFxdnfP311222Xag6WvO2bduMY8eOGYZhGHv37jVuuOEG4/jx491XwHnoaM2GYRjNzc3G7NmzjWXLlhmrVq3qtrl3lM40esA777xDamoqAIMGDWLEiBFs27atVb8///nPxMfHExERgcViYfr06RQUFPjbS0pKKC8vx+12d9vcz1dn1Hzttddit9sBCA8PZ+jQoVRUVHRfEW2oq6ujtLQUl8sFgMvlorS0lPr6+hb9CgoKmD59OhaLhYiICOLj4yksLGyz7ULUGTXHxcVxySWXADBs2DAMw+DIkSPdWkd7dEbNAM899xw333wzgwYN6s7pd5hCowdUVlYyYMAA/88Oh4OqqqpW/TweD9HR0f6fo6Oj8Xg8ABw7downnniCFStWdP2EO0Fn1PxdZWVlfPLJJ/zoRz/qmgmfB4/Hg91ux2q1AmC1WomKimo1//+u8bvPxbnaLkSdUfN35eTkcPXVV9O/f/+unXgHdEbN+/bto7i4mLlz53bbvDvLRf/V6D3htttuo7Ky8oxtf/3rXztljNWrVzNz5kzsdvsZ11O7W3fUfFpNTQ2LFy8mMzPTf+Yhvd9HH33E008/fUFeq+pMJ0+e5NFHH2XlypX+4OlNFBpd4M033zxne3R0NBUVFURERACnXpHceOONrfo5HI4Wf4grKytxOBwA/OMf/2Dbtm2sX7+eEydO8O2335KUlMTbb7/diZWY1x01w6mlgZ///OfMnz+fW2+9tZNm3zkcDgfV1dV4vV6sViter5eampoW8z/dr7KykpEjRwItX5Geq+1C1Bk1A+zatYsHHniA9evXM2TIkG6tob06WnNtbS2HDh1i4cKFABw9ehTDMGhsbOTxxx/v9nraracvqgSitWvXtrgoPHbsWKOhoaFVvzNdFN66dWurfjt37rzgL4R3Rs319fVGUlKS8fLLL3fr3Ntj9uzZLS6Qzp49u1WfP/3pT60ukB46dKjNtgtVR2v+9NNPjYkTJxqffPJJt867Izpa83etXbu2V10IV2j0gH//+9/Gvffea8THxxu33HKL8d577/nbnnrqKeOVV17x//zqq68akyZNMiZNmmQsX77caG5ubnW83hAanVHzqlWrjNjYWGPatGn+/7Zs2dLttZzLV199Zdxxxx3GLbfcYtxxxx1GWVmZYRiGMX/+fGP37t2GYZx618zy5cv9Nb722mv+/c/VdqHqaM0pKSnGjTfe2OL3um/fvh6pxayO1vxdvS00dOc+ERExTe+eEhER0xQaIiJimkJDRERMU2iIiIhpCg0RETFNoSEiIqYpNERExDSFhoiImPZ/AbWuYDe5iZTvAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "x = np.linspace(0, N_EPOCHS,N_EPOCHS)\n",
        "\n",
        "plt.plot(x,t_f1)\n",
        "plt.plot(x,v_f1)\n",
        "plt.title(\"F1 score\")\n",
        "plt.legend([\"Train F1\", \"Valid F1\"])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GAS-PN-mfLJh",
        "outputId": "c5d56c16-aa5c-41b8-ccf7-5e602cbdd0ef"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test Loss: 119.986 |  Test F1 score: 15.53%\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import confusion_matrix, precision_recall_fscore_support\n",
        "model.load_state_dict(torch.load('tut3-model_conll.pt'))\n",
        "\n",
        "test_loss, test_f1, preds, labels = evaluate(model, test_iterator, TAG_PAD_IDX, full_report=False)\n",
        "print(f'Test Loss: {test_loss:.3f} |  Test F1 score: {test_f1*100:.2f}%')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "d4ZpaNA-fLJh"
      },
      "outputs": [],
      "source": [
        "predict =  [item for sublist in preds for item in sublist]\n",
        "true =  [item for sublist in labels for item in sublist]\n",
        "confusion = confusion_matrix(true, predict)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CkQNBI4kfLJh"
      },
      "source": [
        "### Confusion matrix"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Uoh41puOfLJh",
        "outputId": "af483036-459a-4b91-fb52-5dd6a059018b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "          O  LOC  PER  ORG  MISC\n",
              "O     37024  405  180  486   212\n",
              "LOC     859  577  136  143    72\n",
              "PER    1930  174  172  196   145\n",
              "ORG    1289  363  212  294   179\n",
              "MISC    368   70  119  146   121"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-9f0cef16-650a-45f5-b76d-760375392800\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>O</th>\n",
              "      <th>LOC</th>\n",
              "      <th>PER</th>\n",
              "      <th>ORG</th>\n",
              "      <th>MISC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>O</th>\n",
              "      <td>37024</td>\n",
              "      <td>405</td>\n",
              "      <td>180</td>\n",
              "      <td>486</td>\n",
              "      <td>212</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LOC</th>\n",
              "      <td>859</td>\n",
              "      <td>577</td>\n",
              "      <td>136</td>\n",
              "      <td>143</td>\n",
              "      <td>72</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PER</th>\n",
              "      <td>1930</td>\n",
              "      <td>174</td>\n",
              "      <td>172</td>\n",
              "      <td>196</td>\n",
              "      <td>145</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ORG</th>\n",
              "      <td>1289</td>\n",
              "      <td>363</td>\n",
              "      <td>212</td>\n",
              "      <td>294</td>\n",
              "      <td>179</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MISC</th>\n",
              "      <td>368</td>\n",
              "      <td>70</td>\n",
              "      <td>119</td>\n",
              "      <td>146</td>\n",
              "      <td>121</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-9f0cef16-650a-45f5-b76d-760375392800')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-9f0cef16-650a-45f5-b76d-760375392800 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-9f0cef16-650a-45f5-b76d-760375392800');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ],
      "source": [
        "confusion_df =pd.DataFrame(confusion)\n",
        "\n",
        "confusion_df.columns=[i for i in TAG.vocab.itos]\n",
        "s = pd.Series([i for i in TAG.vocab.itos])\n",
        "confusion_df = confusion_df.set_index([s])\n",
        "\n",
        "confusion_df['LOC'] = confusion_df['B-LOC'] + confusion_df['I-LOC']\n",
        "confusion_df['PER'] = confusion_df['B-PER'] + confusion_df['I-PER']\n",
        "confusion_df['ORG'] = confusion_df['B-ORG'] + confusion_df['I-ORG']\n",
        "confusion_df['MISC'] = confusion_df['B-MISC'] + confusion_df['I-MISC']\n",
        "\n",
        "\n",
        "confusion_df = confusion_df.drop(columns=[ i for i in TAG.vocab.itos if i != 'O'])\n",
        "\n",
        "confusion_df.loc['LOC'] = confusion_df.loc['B-LOC'] + confusion_df.loc['I-LOC']\n",
        "confusion_df.loc['PER'] = confusion_df.loc['B-PER'] + confusion_df.loc['I-PER']\n",
        "confusion_df.loc['ORG'] = confusion_df.loc['B-ORG'] + confusion_df.loc['I-ORG']\n",
        "confusion_df.loc['MISC'] = confusion_df.loc['B-MISC'] + confusion_df.loc['I-MISC']\n",
        "\n",
        "confusion_df = confusion_df.drop([i for i in TAG.vocab.itos if i != 'O'  ])\n",
        "\n",
        "confusion_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YLgM8j3ffLJi"
      },
      "source": [
        "### Precision-Recall-F1Score"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "qNIU_9aQfLJi",
        "outputId": "1b5877fb-82d4-4d97-df0c-a4954a9d0f86"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      Precision    Recall  F1-score\n",
              "O      0.892790  0.966507  0.928187\n",
              "LOC    0.363121  0.322888  0.341825\n",
              "PER    0.210012  0.065724  0.100116\n",
              "ORG    0.232411  0.125802  0.163243\n",
              "MISC   0.165981  0.146845  0.155827"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8c246a34-ae8f-4877-bff1-cb80c7beaa20\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Precision</th>\n",
              "      <th>Recall</th>\n",
              "      <th>F1-score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>O</th>\n",
              "      <td>0.892790</td>\n",
              "      <td>0.966507</td>\n",
              "      <td>0.928187</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>LOC</th>\n",
              "      <td>0.363121</td>\n",
              "      <td>0.322888</td>\n",
              "      <td>0.341825</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>PER</th>\n",
              "      <td>0.210012</td>\n",
              "      <td>0.065724</td>\n",
              "      <td>0.100116</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ORG</th>\n",
              "      <td>0.232411</td>\n",
              "      <td>0.125802</td>\n",
              "      <td>0.163243</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>MISC</th>\n",
              "      <td>0.165981</td>\n",
              "      <td>0.146845</td>\n",
              "      <td>0.155827</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8c246a34-ae8f-4877-bff1-cb80c7beaa20')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8c246a34-ae8f-4877-bff1-cb80c7beaa20 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8c246a34-ae8f-4877-bff1-cb80c7beaa20');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ],
      "source": [
        "cm = confusion_df.to_numpy()\n",
        "\n",
        "TP = np.diag(cm)\n",
        "FP = np.sum(cm, axis=0) - TP\n",
        "FN = np.sum(cm, axis=1) - TP\n",
        "\n",
        "num_classes = 4\n",
        "TN = []\n",
        "for i in range(num_classes):\n",
        "    temp = np.delete(cm, i, 0)    \n",
        "    temp = np.delete(temp, i, 1) \n",
        "    TN.append(sum(sum(temp)))\n",
        "    \n",
        "precision = TP/(TP+FP)\n",
        "recall = TP/(TP+FN)\n",
        "f1 = (2*precision*recall)/(precision+recall)\n",
        "\n",
        "prf_df =pd.DataFrame()\n",
        "prf_df['Precision'] = precision\n",
        "prf_df['Recall'] = recall\n",
        "prf_df['F1-score'] = f1\n",
        "\n",
        "s = pd.Series([i for i in confusion_df.index])\n",
        "prf_df = prf_df.set_index([s])\n",
        "\n",
        "prf_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HIbD8H4EfLJi"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "98xiya6SfLJi"
      },
      "outputs": [],
      "source": [
        "def tag_sentence(model, device, sentence, text_field, tag_field):\n",
        "    \n",
        "    model.eval()\n",
        "    \n",
        "    if isinstance(sentence, str):\n",
        "        nlp = spacy.load('en_core_web_sm')\n",
        "        tokens = [token.text for token in nlp(sentence)]\n",
        "    else:\n",
        "        tokens = [token for token in sentence]\n",
        "\n",
        "    if text_field.lower:\n",
        "        tokens = [t.lower() for t in tokens]\n",
        "        \n",
        "    max_word_len = max([len(token) for token in tokens])\n",
        "\n",
        "    numericalized_tokens = [text_field.vocab.stoi[t] for t in tokens]\n",
        "    unk_idx = text_field.vocab.stoi[text_field.unk_token]  \n",
        "    unks = [t for t, n in zip(tokens, numericalized_tokens) if n == unk_idx]\n",
        "    \n",
        "    token_tensor = torch.as_tensor(numericalized_tokens)    \n",
        "    token_tensor = token_tensor.unsqueeze(-1).to(device)\n",
        "\n",
        "    predictions, _ = model(token_tensor)\n",
        "    predicted_tags = [tag_field.vocab.itos[t] for t in predictions[0]]\n",
        "    \n",
        "    return tokens, predicted_tags, unks"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FfDiYjcnfLJi",
        "outputId": "328f755e-9c55-4465-dce9-b87d4dad9110"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Trailing', 'by', '213', ',', 'Somerset', 'got', 'a', 'solid', 'start', 'to', 'their', 'second', 'innings', 'before', 'Simmons', 'stepped', 'in', 'to', 'bundle', 'them', 'out', 'for', '174', '.']\n",
            "['O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
          ]
        }
      ],
      "source": [
        "example_index = 6\n",
        "\n",
        "sentence = vars(valid_data.examples[example_index])['text']\n",
        "actual_tags = vars(valid_data.examples[example_index])['tag']\n",
        "\n",
        "print(sentence)\n",
        "print(actual_tags)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0plwO5f2fLJi",
        "outputId": "459cc53e-ce68-4511-8a8a-1a3f4579ee2a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-LOC', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n",
            "['O', 'O', 'O', 'O', 'B-ORG', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-PER', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'O']\n"
          ]
        }
      ],
      "source": [
        "tokens, pred_tags, unks = tag_sentence(model, \n",
        "                                       device, \n",
        "                                       sentence, \n",
        "                                       TEXT, \n",
        "                                       TAG\n",
        "                                      )\n",
        "print(pred_tags)\n",
        "print(actual_tags)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vPxeCz1BfLJj",
        "outputId": "23e89794-4864-4f4b-bb6a-20d878ec3eee"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pred. Tag\t\t\t\tActual Tag\t\t\t\tCorrect?\t\t\t\tToken\n",
            "\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t Trailing\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t by\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t 213\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t ,\n",
            "O \t\t\t\t\t B-ORG \t\t\t\t ✘ \t\t\t\t\t Somerset\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t got\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t a\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t solid\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t start\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t to\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t their\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t second\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t innings\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t before\n",
            "B-LOC \t\t\t\t B-PER \t\t\t\t ✘ \t\t\t\t\t Simmons\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t stepped\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t in\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t to\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t bundle\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t them\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t out\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t for\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t 174\n",
            "O \t\t\t\t\t O \t\t\t\t\t ✔ \t\t\t\t\t .\n"
          ]
        }
      ],
      "source": [
        "print(\"Pred. Tag\\t\\t\\t\\tActual Tag\\t\\t\\t\\tCorrect?\\t\\t\\t\\tToken\\n\")\n",
        "for token, pred_tag, actual_tag in zip(tokens, pred_tags, actual_tags):\n",
        "    correct = '✔' if pred_tag == actual_tag else '✘'\n",
        "    space = 5 if pred_tag == 'O'else 4\n",
        "    space1 = 5 if actual_tag == 'O'else 4   \n",
        "    print(pred_tag,\"\\t\"*space, actual_tag, \"\\t\"*space1, correct,\"\\t\"*5, token)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install spacy"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r_PIWOHm5DK8",
        "outputId": "eec45ebf-8196-4e61-a5ec-b95f325fc327"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.9/dist-packages (3.4.4)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.9/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (2.25.1)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (1.0.4)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (1.0.9)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.9/dist-packages (from spacy) (1.10.6)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy) (3.0.8)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (8.1.9)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.9/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.9/dist-packages (from spacy) (2.4.6)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (4.65.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (23.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (1.22.4)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy) (2.0.7)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from spacy) (0.7.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from spacy) (3.1.2)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.9/dist-packages (from spacy) (6.3.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.9/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from spacy) (63.4.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.9/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (4.0.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.9/dist-packages (from typer<0.8.0,>=0.3.0->spacy) (8.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->spacy) (2.1.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy download en"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b3Cn105l5LPt",
        "outputId": "2eb529dd-3359-455e-96d0-ad4d91ee2d98"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/torch/cuda/__init__.py:497: UserWarning: Can't initialize NVML\n",
            "  warnings.warn(\"Can't initialize NVML\")\n",
            "2023-03-17 04:47:06.467380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
            "2023-03-17 04:47:06.482064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64\n",
            "2023-03-17 04:47:06.482124: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "2023-03-17 04:47:13.575987: E tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:267] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "\u001b[38;5;3m⚠ As of spaCy v3.0, shortcuts like 'en' are deprecated. Please use the\n",
            "full pipeline package name 'en_core_web_sm' instead.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-sm==3.4.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.1/en_core_web_sm-3.4.1-py3-none-any.whl (12.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.9/dist-packages (from en-core-web-sm==3.4.1) (3.4.4)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.0.7)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (0.10.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.25.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (23.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (1.10.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (63.4.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (3.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (4.65.0)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (8.1.9)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (3.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (3.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.4.6)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (6.3.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (1.0.9)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (1.22.4)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (0.7.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (0.10.1)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (1.0.4)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.9/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (3.0.12)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.9/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (4.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2022.12.7)\n",
            "Requirement already satisfied: chardet<5,>=3.0.2 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (4.0.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (1.26.15)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.10)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (0.0.4)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.9/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (0.7.9)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.9/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (8.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.9/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.1) (2.1.2)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6jQhWvjAfLJj",
        "outputId": "fee756a0-7085-4be8-ee82-ae73d4e97092"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[]\n",
            "Pred. Tag\tToken\n",
            "\n",
            "O \t\t The\n",
            "O \t\t will\n",
            "O \t\t deliver\n",
            "O \t\t a\n",
            "O \t\t speech\n",
            "O \t\t about\n",
            "O \t\t the\n",
            "O \t\t conflict\n",
            "O \t\t in\n",
            "O \t\t North\n",
            "O \t\t Korea\n",
            "O \t\t tomorrow\n",
            "O \t\t in\n",
            "B-LOC \t New\n",
            "O \t\t York\n",
            "O \t\t with\n",
            "O \t\t my\n",
            "O \t\t friend\n",
            "B-LOC \t Mary\n",
            "I-LOC \t Kate\n",
            "O \t\t .\n"
          ]
        }
      ],
      "source": [
        "sentence = 'The will deliver a speech about the conflict in North Korea tomorrow in New York with my friend Mary Kate.'\n",
        "tokens, tags, unks = tag_sentence(model, \n",
        "                                  device, \n",
        "                                  sentence, \n",
        "                                  TEXT, \n",
        "                                  TAG\n",
        "                                )\n",
        "\n",
        "print(unks)\n",
        "print(\"Pred. Tag\\tToken\\n\")\n",
        "\n",
        "\n",
        "for token, tag in zip(tokens, tags):\n",
        "    space = 2 if tag == 'O'else 1\n",
        "    print(tag, \"\\t\"*space, token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "N5vonRgyfLJj"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [
        "k4LF_qyLKvfK"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}