tf_idf_transformation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf_idf_transformation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "wNvgUwVPBKnx"
      },
      "outputs": [],
      "source": [
        "import spacy\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python3 -m spacy download en_core_web_lg\n",
        "nlp = spacy.load(\"en_core_web_lg\")"
      ],
      "metadata": {
        "id": "Sb7e-nc8BPTP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "file_name = 'transformed_data.json'\n",
        "path = os.path.join('.', file_name)"
      ],
      "metadata": {
        "id": "0oYtlruWBalP"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "with open(path) as f:\n",
        "  dataset = json.load(f)"
      ],
      "metadata": {
        "id": "9ciW4-uhCn9Z"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "instance = dataset[0]\n",
        "\n",
        "print(instance.keys())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fIvfOpqNCuqU",
        "outputId": "5a4d0c28-ae29-497e-a75d-1e3a6aa8df32"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dict_keys(['title', 'text', 'url', 'tokenized_text'])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "length_of_documents = [len(doc.get('tokenized_text')) for doc in dataset]\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "plt.plot(length_of_documents)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "88VrGjeWPpBV",
        "outputId": "ae52bb9a-44e9-43fd-c7b3-8e9c6dffd72d"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29e3hjd3Xv/V2632zZsmWPr+MZe3KdSTLJZJKQhEK4NFBKgENbSAuht7Sn8Bx6ejvQvu85tMAppy2lB/qW8wABQh+uh0tJIS2EwHsgJGQyGZKZZO6emYztsSX5Ksmy7r/zx94/WbYla0t767a9Ps8zz8hb0taWZK+99vp913eREAIMwzCMubA0+wAYhmEY4+HgzjAMY0I4uDMMw5gQDu4MwzAmhIM7wzCMCbE1+wAAoLe3V4yNjTX7MBiGYdqKZ599dl4IESx1X0sE97GxMRw9erTZh8EwDNNWENFL5e7jsgzDMIwJ4eDOMAxjQji4MwzDmBAO7gzDMCaEgzvDMIwJ4eDOMAxjQji4MwzDmBAO7gZz9NIiTl6JNvswGIbZ4XBwN5j3f/ME/vZ7p5t9GAzD7HBaokPVTMxFk3DZrc0+DIZhdjicuRtIMpNDLJnF4mq62YfCMMwOh4O7gYSjKQDAwmqqyUfCMMxOh4O7gUTiSQBAMpNHIp1t8tEwDLOT4eBuIDJzB4CFOJdmGIZpHhzcDSQcWw/uXHdnGKaZcHA3kHAsWbjNwZ1hmGbCwd1AwtEUiJTbCxzcGYZpIhzcDSQST2FPjxcAsMiKGYZhKvCLH/sxPvXjybrsm4O7gYSjKezp9cJhtXDmzjDMtqylczgTiiGTE3XZPwd3AwnHUujrdCLgdWCR1TIMw2yDXKPr73TVZf8c3A0im8tjYTWFYIdLCe6cuTMMsw1zKzK4O+uyfw7uBrGwmoYQQLDDiR6fg8syDMNsS0iVTu/izL21kQ1MfR1OztwZhqlIOKpk7n3NCu5E5CKiI0T0PBG9SER/qW7fQ0RPE9F5IvoqETnU7U715/Pq/WN1OfIWQ1oPcHBnGEYLcytJuOwWdLrqY86rJXNPAbhHCHEjgJsA3EtEtwP4HwA+JoSYALAE4LfVx/82gCV1+8fUx5meQube6UKP14F4KotkJtfko2IYplUJxVLY1ekCyeYYg6kY3IVCXP3Rrv4TAO4B8HV1+8MA3qTevk/9Ger9r6J6HX0LIa0Hgj4nAl5lgYSzd4ZhyhGKJutWkgE01tyJyEpEzwEIA3gMwCSAZSGEtD6cBjCk3h4CMAUA6v0rAHpK7PNBIjpKREcjkYi+d6GRRDqL3/r8M7g4v2r4vsOxJLo9djhsFgS8DgAc3BmGKU8omqybDBLQGNyFEDkhxE0AhgEcBnCN3hcWQnxKCHFICHEoGAzq3Z0mzszF8MPTYTxxzviTSTiaQrBDydh7fEpwZ8UMwzClEEIowb2jPjJIoEq1jBBiGcCPANwBoIuI5ErAMIAZ9fYMgBEAUO/3A1gw5Gh1Mq82Fs0sJys8snrCsRT6OpSz8HrmzhYEDMNsJZrMIpnJY5e/iZk7EQWJqEu97QbwGgCnoAT5t6oPewDAt9Xbj6g/Q73/h0KI+vTXVsl8XAm2sytrhu87EkuhT2buanBnT3eGYUpRbxkkoG1A9gCAh4nICuVk8DUhxHeI6CSArxDRhwD8HMBD6uMfAvDPRHQewCKAt9XhuGtiXl30nDU4cxdCIBJLIah2mnW67LBZiGvuDMOUZE4N7vUsy1QM7kKI4wAOlth+AUr9ffP2JIBfMeToDEZm7lcMztxX1jJI5/KFsozFQuhmrTvDMGUIqdLpppZlzISsuYeiSeTzxlWKCjLIorNwj5ctCBiGKU1IlmU6OLgbQkTN3DM5UcjijaDYekDCXaoMw5QjFE2i02WD22Gt22vsqOA+H0/BbVc+zCsrxtXdi60HJBzcGYYpRyiarGtJBthhwX0hnsb+oU4AwJVl4+ruxdYDkh6vAwsGXh0wDGMeQtFUXRuYgB0U3NPZPFbWMjgw1AXA4OAeS8HjsMLnXF+fDnidiCazyOTyhr0OwzDmIBRN1rXeDuyg4L6gNhRN9Pngtlsxa2BZJlykcZcE1C7VJS7NMAxTRD4vEI6lsMtfPxkksIOC+3xMCbK9PgcGulyGNjKFo8kNShmgqJGJgzvDMEUsrKaRywsuyxiFVMf0djgx6HfjioGNTJEi6wEJm4cxDFOKRsgggR0U3KUMMuhzYsBvbOYeiaU4c2cYRhMyuLNaxiAKmbvPiYEuN8KxFNJZ/Yuda+kcYqks+jYNuS1k7qyYYRimCNmdWq/B2JKdE9xjaXgdVrgdVgz6XRBi/Qyqh3Cs9CVWl8cBIi7LMAyzkVA0CSIl0awnOye4x1PoVUsng11uADBEMVPKegAArBZCt4ctCBiG2UgomkSP1wm7tb7hd2cFd58M7kqWbUTdvZT1gIS7VBmG2YzSnVrfrB3YYcFdLnIO+JXM3QjFzHpZpnRwZ093hmGKCUVT6K+zUgbYUcE9XSjLeJ02dLpshmTukVgKNrUEsxnFGZIXVBmGWafeg7ElOyK4Z3N5LCXSGxYwBruM0bqHVRmkxUJb7uOyDMMwxaSzeSysprGLg7sxLK6mIQQQ9K1n1wN+lyH+MuESGndJj9eB5bUMcgZ6xzMM077Ifpt6yyCBHRLcI0Uad8lgl9ugBdVkyXo7oGTuQgBLCc7eGYYB5lSFXr2tB4AdEtzlBKbejo3BfSmRwVo6p2vfSndq6S8qoJ5MuDTDMAywPhibg7tByMHYxZn7gF+/HDKTy2MxkS6buRcsCFgxwzAM1hsnuSxjEFKx0ruh5q6/kWkhrtTyN1sPSHp8bB5mBP/fj87jDZ/4SbMPg2F0E4qlYLeWVtcZzY4I7vPxNJw2y4ZhGrKRSc+iajnrAcm6MyTLIfXwwswKXpiJIpnRV0JjmGYTWlGGdJRS1xnNzgjuMaU7lWj9A5WObHrkkLI7tZxaRp6d2YJAH9LiYXrJOCdPhmkGoViyISUZQENwJ6IRIvoREZ0koheJ6L3q9g8Q0QwRPaf+e33Rc95PROeJ6AwR/WI934AWIkW+MhKnzYpen1NXzV0GnXI1d7vVAr/bzmUZncgrpKnFRJOPhGH00YjZqRJb5YcgC+CPhRDHiKgDwLNE9Jh638eEEH9X/GAiug7A2wBcD2AQwA+I6CohRNOuqefjaQx1bf1AB7tcuKKj5i6DznbubkqXKgf3WhFCFK6QLnNwZ9qc0EoSd030NuS1KmbuQohZIcQx9XYMwCkAQ9s85T4AXxFCpIQQFwGcB3DYiIOtlWLTsGIG/C7M6qi5R2IpBLwOOGzlP8aA14FFVsvUTDSZRUr13efMnWlnVlNZxFLZhmXuVdXciWgMwEEAT6ub3kNEx4nos0TUrW4bAjBV9LRplDgZENGDRHSUiI5GIpGqD1wr+bzA4mq6THB361LLlBqMvRm2INBHJLa+GM2ZO9POyDJuy9TcJUTkA/ANAH8ohIgC+CSAcQA3AZgF8NFqXlgI8SkhxCEhxKFgMFjNU6tiKaEMoy2WQUoGu1yIp7KIJjM17Xs76wFJj4/LMnqQpa8Op42DO9PWNLI7FdAY3InIDiWwf1EI8U0AEEKEhBA5IUQewKexXnqZATBS9PRhdVtTKNWdKilo3WtUzESiyYrBPeB1YCmRRp79ZWpCZu43jXZhemkNQvDnyLQnMlFpmeBOin7wIQCnhBB/X7R9oOhhbwbwgnr7EQBvIyInEe0BsA/AEeMOuTrmS/jKSPRo3YUQiMRTFSeYB7xO5PKi5quDnY5cTL1ldzfiqSyWEvw5Mu1JI7tTAW1qmTsBvAPACSJ6Tt325wDeTkQ3ARAALgH4PQAQQrxIRF8DcBKK0ubdzVXKbBfc1aEdNcghlxIZZHKiYs29YEGwmkaXAV1pQgiksnm47Fbd+2oHwrEknDYLrh/0A1AWVWVzGMO0E3MrKXgc1g3NlPWk4qsIIZ4AUKqd6tFtnvNhAB/WcVyGEYlttR6Q9HW4YLVQTWUZud9y1gOS9S7VNMYNWFr430en8eFHT+GJ//JKdLjs+nfY4oRjKfR1OjEa8ABQFlVvHOlq8lExTPWEYkns6nRtaKasJ6bvUJ2Pp2G3EvzurYHQaiH0dzhrytwrWQ9IAgabhx2fWcbKWgY/u7BoyP5anUhMKX2NBJSrLF5UZdqVcDRZMRk0kh0Q3FPo8TrLni0Hutw1Ze6VrAck0jzMqHF7U4vKiegn5+onH20lwrEUgj4nPA4ben0OTC9xcGfak0Z2pwI7JLj3dpSv0Q74XTVZEFSyHpAUyjIGZe6ykeeJc/OG7K/VKc52RgIeztyZtkQIgblosiHj9SQ7I7hvYw8w2OXGlZVk1RK7cCwJr8MKb4XFEadNWUAxQuuezwtML62hw2XDhflVzBgwJrCVSWZyiCazhRPoSDcHd6Y9WVnLIJ3NN2QwtsT8wT1WujtVMuh3FYbWVkMkltL8RRnVpRqKJZHO5fGWg0rD7xMmL80UFq3VdY3RgAdXlpPI5vLNPCyGqZpQtLHdqYDJg7sQAgur22fuA121NTJp6U6VGBXcLy8oWes91/ajv9OJn5i8NCNLX0H1D2I04EEuL3RZRjBMM5hTNe5cljGI6FoWmVxp6wHJoL82rXukiuBulDPklOpnPhrw4M6JXjw5uWDqztdIQZGkfM7DqmKGDcSYdiPUwNmpElMH90i8sqJlQO1SrdYdMhxNVlxMlSiZu361zOXFBIiAoS437t7Xi8XVNE7ORnXvt1UpZO4d65k7wHJIpv2Qg7G1JoRGYOrgvl13qqRHteyt5lJ/NZXFajpXUeMuCfiUsoxeX5SpxQQG/W44bBbcqXpC/9jEdfdwNAULAT1e5fsb8LthsxAHd6btmIsm0e2xN7SzfMcHdyLCgL+6oR1aZZCSHq8DmZxALJXV/BqlmFpMYLjbrb62C9fs6jC1JDIcS6LX54RVnTdptRCGut2F8hTDtAuN1rgDZg/u21gPFDPod1dlHqbVekASUDNPvVr3y4uJQmkCAO6a6MXRS0tYS5tzcLS0HihmlLXuTBui9GtwcDeM+XgaVgsVBlWXY6CruolMWq0HJOtdqrUH92Qmh3AshZGi4H73VUGkc3kcuWROKwJpPVDMcLcH0xzcmTZjLppEfwPr7YDpg7syBs9i2d6oZ9DvRiiWQk6j8kSr9YCkp8g8rFZk231x5n54LACH1WJavXupSVejAQ8WVtOI6yxxMUyjyOUFIrEUdvk5czeMSt2pkoEuF3J5UcjIKxGOpWC3Ero92lwZ150ha1fMyFJEcebudlhxaKzblHr3XF5gIb5VbjrCckimzViIp5AX4LKMkUTi6Yr1dqBI666xkSkcSyLoK29Gthmp9tBTlpGGYTK4Se7a14vTczHNJ6Z2ofAHUSJzBzi4M+1DoTuVyzLGMa86ClaioHXX2MgUiaUQrOIs7HZY4bZbdS2oXl5MwGW3bHk/d08oJvE/PW+u7H1d477xc2atO9NuFLpTuSxjDEII1RFSQ3AvZO7ag7tWGaRErwXB1GICI92eLVcL1w92ottjN11pprBovUkt43fb0eG0cebOtA3N6E4FTBzc46ksUtl8YTFzOzpdNvictirKMtqtByQ9Pn0WBJtlkBKLhfCyiV48cW7eVMOjI2V6CYgIIwEPa911spbO4d9OzDb7MHYE4WhSbcZr7HhI0wb3ebUEomVBVTYyaSnLpLN5LK6mG5q5CyGUzL1EcAeAl+/rRTiWwrlwvKb9tyJSkVTq+xsJuLkso5PvHL+C//jFYzgbijX7UEzPXDSJYIcTNmtjw62Jg7saHDQG4YEutyYLArlfrRp3iZ7gvpTIYDWdKxvc79qn1N3NVJoJx1Lwu0u3a48GPJhaTJjqSqXRyN91Du71pxndqYCZg7vG7lTJoN+lqSxTrfWApMfrwHw8VVNAkllqqbIMoBiJ7e31mkrvHo6VN2YbDXiQyuYLpRumeuSaxrmQea72WpVQNFl1MmgE5g3u0hFSQ1kGUBZV5+MppLLbt/JXaz0gCXidSGXzSNRgFTBVIbgDiiTyZxcWKx5/u1DKekAyzIoZ3Uh53vkIB/d6E4omscvfWBkkYOrgngbRegNRJaQcMrSyfTYoM56qF1R1dKnKICZNw0px10Qv1jI5HHtpuer9tyKlrAckBa07D8uuGXkFOmmidZpWJJXNYSmRQT9n7sYxH0+h2+PQvIgxpE5kqjSXNBxNgUjbQm0x8iRTi2JmajGBXp9j23mtt4/3wGohPHG+/UszQoiS1gOSoS43iIDLC6yYqZWIKs+7ML+q2XaDqZ5wYbxeCwZ3Ihohoh8R0UkiepGI3qtuDxDRY0R0Tv2/W91ORPRxIjpPRMeJ6OZ6v4lSKNYD2qVHA35tjUzhWAoBjwP2Kle+A77aLQimlhIY7i5fkgGATpcdN410mcICOLqWRTqbL3t15LJb0d/h4rJMjQghEFGtOdLZPPcM1BF5pd/f4AYmQFvmngXwx0KI6wDcDuDdRHQdgPcBeFwIsQ/A4+rPAPA6APvUfw8C+KThR62B+fj2g7E3IxuZKilmIrFkTdNUZFlmoYYu1XIa983cva8Xx2dWsJzQP9KvmWgpfY0GPFyWqZGlRAaZnMDLxnsAAOfrUJoRQuBD3zmJ49PmKBPWytxK4wdjSyoGdyHErBDimHo7BuAUgCEA9wF4WH3YwwDepN6+D8AXhMLPAHQR0YDhR14BraZhErfDim6PvWKXqrLQV/1ZOFBjzT2by+PKclJzcBcCeHJyoerjayXWFUnlP+fhgJszzhqRHZOF4F6HRdUrK0l85omL+OaxGcP33U4UulNbveZORGMADgJ4GkC/EEK2uM0B6FdvDwGYKnratLpt874eJKKjRHQ0EjG+Tjwfqy64A0r2Xjlzr956AAB8ThscVkvVwX12JYlcXmwxDCvFjcNd6HDa2l7vXs56oJjRgAdz0aRp1EGNRJ48J/p8CHY465K5n5lTZvtO7nA1TiiWhMNmQZdGB1kj0RzcicgH4BsA/lAIsWEqs1DE21WtygghPiWEOCSEOBQMBqt5akXW0jmspnPo7aiu3Xewy7Vt5p5XfZlrKcsQEQLe6i0ISln9lsNmteD28Z62X1QtZz1QzGjAAyGAGbYhqBo5rLmvw4WJoK8uwf3UrNIcdSGyavi+24nQShL9ndodZI1EU3AnIjuUwP5FIcQ31c0hWW5R/w+r22cAjBQ9fVjd1jC0zE4txWDX9uP2lhJpZPOipswdqK1LVYvGvZi79/VianENLy207x9VOJqC226Fbxt10Ahr3WsmXNSrMdHnw2Q4bni37+k5JbjPLK+ZdgykFkLRVFNKMoA2tQwBeAjAKSHE3xfd9QiAB9TbDwD4dtH2d6qqmdsBrBSVbxpCpMoGJsmA341oMovVMlN+tNSCt6MW87DLiwnYLFRY8K3EXRO9AIAft3FpRhqzbZftrGvdOXOvlnA0iQ6XDS67FRN9PsRS2cLvtlGcmYvCoSrKLszv3NJMKJZsigwS0Ja53wngHQDuIaLn1H+vB/ARAK8honMAXq3+DACPArgA4DyATwP4A+MPe3vWrQeqzdy3l0OGa+xOlSiZe3V/RJcXExjqdsNaYVSgZE+vF0Nd7ra2ItjOekAS9DnhsFl4UbUGwrF1r5OJPh8AYxUzqWwOk5FVvPwqJdGY3MGlGaUs05zgXv66V0UI8QSAcpHlVSUeLwC8W+dx6aLgCFllzX2gaCLTRF/HlvvXa5U6gnuVUsippTWMVNC4F0NEuHtfL757YhbZXL7hTnRGEI6lcM2urZ9/MRYLYaTbjcsLHNyrpbhBrDi436le9enlfDiOXF7g3v0DePx0GBfqtKj6xLl5ROJJvPngcF32r5d4KovVdK4pMkjApB2qsuYux9tppVIjU6HcU2Nw7/U5sZrOIZnRXoPczuq3HHft60UsmcXxmZVqD7EliETLWw8Uw1r32ii+MurrcKLDaTM0cz+j1ttvGvFjuNtdt8z9Ez88h4/82+m67NsImjWkQ2La4O532+GwVff2dvldICo/SzUcTcHntMHjqHjBU5Jqte7xVBaLq2nNi6mSO8d7QYS27FZdS+cQS2U1nUBHAh5eUK0SIQTC0fVeDSLCeJ+xipnTczE4bBaM9XgxHvTVxb9GCIGzoRhC0VRVyVIjCa1wcDec+XgKPVVYD0jsVmVGadnMvUaNu6Ta4D5VkEFqW0yVdHsd2D/ob8vgrkUGKRkNeBBLZrGSyNT7sExDdE2ZUFb8+U70+QxtZDo1G8W+Ph9sVgvGgz5cnF9F3mD/mvl4Gkvq9z7doovqIWk9wGUZ45iPVWc9UIwihyyTuddoPSDpqdI8rJKP+3bcta8Xxy4vIV5G+dOqVOO6Kf12OHvXznqD2Ho2OdHnQySWwsqaMSfJM3MxXLOrEwCwN+jFWiaH2ai2EZZaOVc0ZKRVS3OhJpqGAWYN7vFU1TJIyWCXC1e2UcvUYj0gWc/ctSlmCpl7FQuqkrsnepHNC/yszawIqpGbjrLWvWpKDZuZCBqnmFmIpzYsiI+r+za6NHOmOLi36PcfiibR4bRt6+ZaT0wZ3CNVOkIWM+B3Y3Y5WbKpQ29ZRi7wajUPm1pMoMNpq6l1+ZaxbrjsFjxxvr1KMwVFkoZLWVmuatXMrRUpZO6byjKAMQFYLqZeM7ApuBusmDkbiqHLY4fL3rpy2FA0WbNs2ghMF9xT2RxiyWzNZZkBvwtrmdyWS9R4KotEOqerLNPptsFmIe0196U1jAQ8NbUuO21W3LanBz9pM717JJ6CzUIIeCqfnDtcdnR77Jy5V4H0Fy++Ah0JeOCwWQypu8vOVFmW6fU50OGyGW5DcDYUx1X9HRjubt1F9WbNTpWYLrgvFDTutdfcga2KGb0ad0BRJnR7HZoz98uLiaoXU4u5e18vJiOrFT3qW4lwVDF8s2hs2pLDshlthKIpeB0brR2sFsLeXq8hZZnTc1H0eB2FJIiIFMWMgZm7EAJn52K4ur8DowEPLi+25u93KJrELg7uxlGrr4yknNZdr/WApEejeZgQAlMafdzLcdc+pSmlnVwit5udWophDu5VEY4lS64bGSWHPD0XK5RkCvs2OLjPRZOIpbK4qt+H0YAH04sJw71x9LJZctoMTBzca6u5DxUy9zLBXWcNTasFQSSWQiqb1xXcr+7vQLDD2VaSyHCsusXw0YAHM8trPCpOI+EyrqYTQR+mlhK6NOO5vKI9lyUZyd6gF6FoyjDllqzrK2UZN2KpLJZbTA67lMggncs3TQYJmDG4x9SyTI2Ze6/PCbuVcGXF+LIMoN0ZsjAUW0dwJyLcNdGLn56fN1xnXC8iseoWoUYDHmRyAnMGS+3MSjlRwESfD0Los+h9aWEVyUx+i3WEXFQ1yobgbGg9uLeqYkp2p3JZxkD0WgRYLIT+ThdmN2XukXgKDqsFfrc+032tZRmpANGTuQOKS+TCarqw0NXKZHN5LKymEayi9CVlouwxo41QNFmytFjwmNERgM9sWkxd37cXgHGKmbOhOIIdTnR7HQVrjlZTTM1Ft/YTNBrTBff5uGIR4LJba97HoN+9JXOPRCvb0Goh4HUillQGQG/H5QXl5CLLRLVyy+5uAGiLWZYLq2kIUd3VUcH6t8Uyt1ZEKr5KlQr29HphIX1a91NzMVgI2Nfv27B9NOCF1UKGKWbOhpTFVKB1ff3D0eZ2pwKmDO7pmuvtkoEuV8kFVT0ySElAPbalCkOsLy8msKvTpeskBQC7ezzocNlwog1MxAoyvSo+54EuFyzUeplbK7JdD4HLbsVIwKNL6356Noo9vd4tv7MOmwWjAY8hmXs+L3AuFC+cQHxOG3q8Dky1mGImFDVGgKEH8wX3GmanbmbA78bcSnJDnVqLx7gWChYEFeSQU0v6lDISIsL+QT9eaIfgXqI1vhJ2qwWDXe6Wy9xakUqKL70j986UWEyVjAe9mAzrz9ynl9awlskVMnegNRVTc9EkeryOqs0LjcR8wT2uP7gPdrmQyQnMF6laqpXolUOredjUYgLDOjTuxRwY9uPUXAyZ3PaloGYjg0+1V0isdddGKeuBYib6FJOvbA2/J6upLF5aSJT14d8b9OHiwqpuVZO0Hbiq6HVGW9AdNBwtLTltJOYM7lUO6djMoH9jI1Mqm8NyIoOgT/+XtW4eVl4OmcrmMBdNGpK5A8D+IT/S2XxBZdCqyLJMtb5AI92t28jSShQPxi7FeJ8P6Vy+ptGFMuheM1A+c09n87oHmsvf4X1963X90YAy+7iWk1K9ULpTm1dvB0wW3DO5PJYSGf1lGTluT1XMyMlOjcrcZ5bWIIR+pYxk/6DyB9fqpZlwLIluT/U+/KM9HszHU0ik28sBs9GEYyk4bRZ0uksbWekZubeulCmduRvlMXM2FMNQlxsdrnXV2ki3B9m8wOxK68hhm92dCpgsuMuAqbssIzN39ZfFKI07AHR5HCDaPrhfLvi4GxPcx3q88Dlbf1FV0WBX/wcx3K18X63q690qhFUjq3KKLz3B/fRsFD6nray6a69Bwf3MXKyEGqe15JDZXB7z8eZ2pwImC+6RGgdjb0a6zcnM3SjrAUDx8ej2bK91l5fFRmXuFgvh+sFOnJiJGrK/elHrukahkYW17tsSrnDy7HTZ0d/prCm4n5qL4epdHWU9gQJeB7o9dl0j97K5PC5EVjcspgLrSVCrrLvMx9PIi+bKIAGTBXe91gMSIsKg3124zDPKekBSaVD21GICDpulZk/6UhwY8uPUbLSlF1UjVVoPSFq1S7HVCGuwrJ7o8+F8uLq1GSEEzqjBfTv0esxcWkggncvjqk3BfcDvgtVCLfP9t0J3KmC64G5MWQZQ6u5yaEckmgTR+mKoXipZEFxeSGCk263ZGVELB4aVRVUjZ2UaiRBCCe41nEADXgc8DmvLXJa3Kkp3aoXgHvRhMrJalRHXXDSJlbUMrq0Q3PcGvboamc4V2Q4UY7NaMNTlbhmt+1yTB2NLTBbc1czdgNr4oN9dMA+LxFPo8TpgsxrzcfX6HNuqZYzSuBezf8gPAC1bd19WjZZqKX0REWtxE0kAACAASURBVMshK5DMKHMOKtWBJ/p8iKeyVXn1nJ7dXikjGQ/6MB9P1Tzz9kwoBqL1tYFiRgKt0+tQzcCZelIxWhHRZ4koTEQvFG37ABHNENFz6r/XF933fiI6T0RniOgX63XgpViIp+CyW+B16OvqBICBLjfCsRQyuTzC0VRVfieVqJi5LyYMW0yV7FEXVVtVMSM9gWpdtB4JeFomc2tFtHb/jtewqHp6rnRGvWXfclF1vrarx7OhGEYDHrhL/H230sk9FE3BaiH0els8uAP4PIB7S2z/mBDiJvXfowBARNcBeBuA69Xn/BMR6Y+0GlGsB/T7vwDAoN8FIZRLWS21ymoIeJ1YXsuUbOhYSWQQS2YNz9wtFsJ1g50tm7nXYj1QzIg6kafVfL1bBa3dv7UoZk7PRTHU5a5oqrc3qBiI1VqakdOXSjHc7cHCahqrLTAQfk4tfxlZVq2FisFdCPFjAIsa93cfgK8IIVJCiIsAzgM4rOP4qsKI7lTJgCrpml1JGmY9IOnxOiBEaX+ZgtVvDUOxKyEXVVup2UNSi/VAMaMBN9YyucK6C7ORSt2pkqDPiU6XrbrgPhsrq28vZiTggd1KNS2qprI5XJzfqpSR1EMOeTYUwy9/4gmcvFKdyizUAt2pgL6a+3uI6LhatulWtw0BmCp6zLS6bQtE9CARHSWio5GIMXM+Iwb4ykgG1YlMM0trmI+nDa2fBbbxl5HB3ejMHVCCezKTN2RWptHUaj0gaVXr11YhpHGRj4hUxYy235F0No/JSLyiUgZQfIB293hrMie7OK9YF2zWuEvW3UGNK839nzMRnJhZwQOfO1JVySccTWFXk+vtQO3B/ZMAxgHcBGAWwEer3YEQ4lNCiENCiEPBYLDGw9jIfDyNoE7rAYnM3F+8soJcXhgqS9zOgkAGJz2zU8tRWFSdbr3STDiagmfTbM9qYOvf7QnHUrBbCd2eyvMIJvq0SxYnI3Fk86LiYqpkb68XF+arL8vIDthyJ5F6WP+emoui02VDOpvHA589onmw/Vw02XSlDFBjcBdChIQQOSFEHsCnsV56mQEwUvTQYXVb3cnlBRZXjcvcfU4bOl02PD+lBEIjL7Ok7W+pX5bLiwl0e+wb2quNYm+vF16HtSUXVfWWvoZ5aMe2hKNKD4GW9aiJPh/m42ksV7ClBpR6O4CKMkjJeJ8PLy2sVt1vcTYUg81C2NtbOnPv9tjhc9oMPbmfmo3h4Gg3HnrgEGaW1/Cbn3+mosVFMpPDylqmfYM7EQ0U/fhmAFJJ8wiAtxGRk4j2ANgH4Ii+Q9TGUkLpCjMquAPAYJe7sABp7IJq+eCudyj2diidqv6WXFSt1XpA4nZYEexwclmmDOFYEkGNAaeaRdXTczE4rBaM9Xo17Xs86EMmJ6oOwmdDcYz1esv6DhGRqpgy5vtXekJiuHagE4fGAvjE2w/ixPQy3v3FY9uemPQKA4xEixTyywCeAnA1EU0T0W8D+BsiOkFExwG8EsB/BgAhxIsAvgbgJIB/B/BuIUTtE3erYL071bgPdcDvwpo6MNhI0/1uT/ma+1QdZJDF7B/y42QLLqrW2sBUTCtav7YK4ah2xddEUMnCNQX32Rgm+nywa+wBqVUxUzx9qRwj3cZp3S/Mx5HJCVw7oLzma6/fhQ+96QB+dCaC933jRFlVVkgVBuzyt0HmLoR4uxBiQAhhF0IMCyEeEkK8QwhxQAhxgxDijUKI2aLHf1gIMS6EuFoI8W/1Pfx11gdjG1NzB9br7oCxDQl2dRbr5sw9lxeYWV6ra3A/MNyJZCavy+OjHhghNx3pbp0uxVYjHEtq9joZ6nbDabNozNyjuGZAW0kGAMZ7qzcQW0vncHkxUXYxVTIa8GBqyRg57KlZtdxUtJZw/22j+MNX78M3jk3jb793puTz5lZaozsVMFGHqpHdqRKpmOlw6ZvJWoqeEo1Mc9EkMjlRt7IMoChmgNbqVE2ks4insrrHGI4GPJhdWas4n3ankc4qVtharz6tFsLeoK+iqmppNY1QNKVJBinxe+zo9TmrCu7nw3EIgcqZe8CDZCZfaIjTw+lZpdy0Z1O56b2v2of7bxvFP/3/k/j8Ty9ueV5BldTE8XoS8wV3Q8sySuZuxOzUzQS8Wy0I5GLgSB007pI9vT54WmxRNWzQvMmRgAd5gYJtBKNQS/evFjnk6YKHuzaljKRaj5lS05dKYaRi6uRsFPv6t5abiAgfvG8/XntdP/7yOyfxneNXNtwfjild8uU88xuJaYJ7JJ6Cw2pBp8u4D1UO7ajH4kgpCwK5GFjPzN1asP9tneCu13pAwlr30tTidTIR9GFmeQ1r6fJLZlIpU01ZBqjeHfJcSMmid1f4uxgxUOt+eq78PFirhfDxtx/Eod3d+KOvPo8nz88X7ptbUWSQRnTJ68U0wX0+lkavz2HohyoHD9RjgnmPr0RwX0zAaqHCSaVe7B/y4+SVqO55lkZRyNwNWFAF2Pp3M6Earowm+nwQYvva+Jm5GAJeR9U9IONBL5YSGc268TOhGMb7fBWN++TQFr3f/3w8hUgsVVhMLYXLbsVn3nkrxno9ePCfn8WLV5RkKRRNtkRJBjBTcI+nDK23A8qKN1F9TPcDXgeWEhnkiwLs5cUEBvwuzcqDWjkw5MdaJqd7Ko5RFKwHdP5R9He6YLe2jq93qxCJ1ZC591Ve+Dw1p9gOVJtQVTty71wojqsqLKYCSsDt73Tq/v6ly+V1FRqz/B47Pv+bh9HhsuFdn3sGU4sJhGMp9LeAUgYwW3A3sN4OAE6bFf/zbQfxzjvGDN0voJiH5fICK2vr9qf11LgXc6DFOlVl92RXBeOpSlgthOFuD6ZZMbOBcCwFCwE9VbgUjvV6YKHycsh8XuDsNqWL7ZDB/YKG4B5LZjCzvFbRcVIy0q1f6y6VMlq6bge73PjCbx1GOpvHOz97BLMra+hvAY07YLrgbpwMUvLGGwfrIk1ctyBYvzS9vLjWkOC+N6gsqrZK3T0cVU7MRrjojbDWfQvy87VW8fk6bVbs7vGWDe6XFxNYy+SqUspIhrrdcNgsmuS4Z0PK61dSykiMsP49NRdFf6ez0GxYiX39HXjogUO4sryGZCbfEjJIwCTBPZ8XWIin0WNw5l5PNnepJtJZzMdTddW4S6wWwnUDnS2jmDHSdXOk280LqpsIx5I1rWeMB8srZmpdTAWU3789Gg3Eyk1fKsdIwIPZaFKXHPbUbPVXJIfGAvjH+2+GzUIV9fiNwhTBPZrMIJsXhpdl6sl6cFcWu6bVodiNCO6Asqj6YossqkZixg1DGQ14sJzIbCh37XRC0dqsHSb6fLi0sFqym/nUbAwWAvb1VR/cAWC8T5uB2JlQDG67tbBYWomRgAdCADM1ymGLbQeq5TXX9eP4B16LV1zdV9NrG40pgrtRg7EbSY9vY1lmXeNuvBtkKeSiqpa6Z72JxFKGdQCPsDvkFsKxVE2igIk+xQfmpRKf5Zm5GMZ6vCWnImlhPOjD5cUEUtnt3UnOheLY1+/TXLLTq5jabDtQLR5H8/XtElME94hqPWCkLW+9KWTuqr9MIzTuxRwYVhZVX7jS3NJMJpfHwmrasLKM/PymuTQDAMjm8lhYre3KaDsDsWptBzazN+hFLi8qunieCcU0l2QA/Y1MpWwH2hVTBPd6WA/UG6dN8S4vZO6LCXgdVs2LOHrZ2+uFy27BienqpswYzXzcmO5UST18vduZhdU0hKitQWxcNfnaHNwT6SxeWkzUpJRZ37eUQ5YvzSytphGJpTQvpgLK+3TYLDUH93K2A+2IuYJ7G2XuwMYuVekG2ajONpvV0hKLqrKBySiLB7/bjk6XjQ3EVKTXSS3BvcNlx65O15aFz7Mh1eulBqWMZK8GrftZdTG1mgVKi4UwrGNRvZztQDvS/u8ASnC3WvTrpBtNcZfq1GJ93SBLcWDIjxevrGxopGo0Wmd7VsNoD8shJfLkWas8b6Jvq4HYaVm60JG5+5w29HdubyAmg3u1JxE5LL0WtrMdaDfMEdxjafR4HU2fNl4tPV4H5uMpCCFweTFRV8OwUuwf8mM1natp7JlRRGLGWA8UY0Qji1kI6/x8J/p8mAzHN9jonp6LwevQrmApx3jQt62B2NlQHB0uG3ZVeWIaDXhqmsilxXagnTBHcK9Dd2ojkGWZhdU01jI5jNZhbup2FBZVm1iaCceSIDK2pDYa8GB6aa2pVyStgt7Pd7zPh9V0DrOqTzmgLKZetatDdzIlDcTK+a/LxdRqS5WjAQ+iySxWEtXJYbXaDrQL5gnubbSYKgl4nVhKpPGSmmWM9jQ2c58I+pRF1aYG9xQCHoehNc6xXi/SuTyXZqD/850IblTMCCEMK13sDXoRS2ZL+q8LIXC2SqWMRA6Xr7buXo3tQDtgkuCebiuNu6TH60AmJ3BS/aVqdFnGZrXg2oHm2v+GoynD/fJvHu0GADxzadHQ/bYj4WhS1+e7WQ4ZjqWwnMgYUrpY95jZWpqJxJXX0WIYtplaFVPV2g60Om0f3IUQiMRTbaVxl8hfoucuLwMAhhsc3AFlUfXklWjTShiRmL7gU4p9fT50eewc3CEbmGqXmfb6HPC77YVFVZndViNPLMf4Ns6TZ+eq85QpptZGtlpsB1qZtg/usVQW6Wy+PWvu6tXG89PL6Otw1tztp4f9Q37EU1lcXGjOomokVltr/HZYLIRDuwM4crF+wf3bz81gwYBxbvWmmsHYpSCiDVOZap2+VIqBThdcdgsmw1t/985qnL5Uik6XHV0ee1WZeyZXu+1Aq9L2wX0+JhuY2u9SSjpDTkbiDZdBSqT9bzMWVeVVl5FKGcnhPd24tJAoeMUbyWQkjvd+5Tl89LGzhu/bSPJ5gXkDPt+JoK+gdT8zF8Og3wW/R7/s2GIh7O314cJ8icw9pAwCqTVpU4Zla+91mIzosx1oRdo/uKvt+22ZuavBXYjG2Q5sZl+fD06bpSne7kuJDDI5UZcxhreOBQAAz1xcMnzfcqzat38+g3gqa/j+jWIxkUY2L3RfGU30+bCwmsbiahqnZqO6mpc2M95XeuSeopSp3V2xWjmsmWwHJCYI7u3ZnQpsHJ7QKMOwzTRzUdWoCUyl2D/kh9tuxZGLC4bv+8nJBbjsFqymc3jkuSuVn9Ak9HSnFiMXVU/PRjEZiRuqJtnb68X00hqSmXUDMSGEOn2p9pPISMCD6aWEZtdTM9kOSCoGdyL6LBGFieiFom0BInqMiM6p/3er24mIPk5E54noOBHdXM+DB9o7uLsdVrjtSp29WWUZQHaqNn5R1WjrgWLsVgtu3t2FI5eMzdzzeYGnLizglw4M4ppdHfjSkZcM3b+RrDcw6c/cAeD7J0PI5ERNAzrKMa7Oar1UtOZzZSWJeCqrK7iPBjzI5EThBFcJM9kOSLS8k88DuHfTtvcBeFwIsQ/A4+rPAPA6APvUfw8C+KQxh1me+VgKRGhb+ZI87maVZQAluMdT2Q1/YI2gHtYDxdw6FsDpuaih3u4nZ6NYTmRw50QPfv22UbwwE8Xx6WXD9m8kkagxn+9QlxsuuwWPnpgFYMxiqkSakxUvqtZqO1CM1LprXVQ1k+2ApGJwF0L8GMBm2cF9AB5Wbz8M4E1F278gFH4GoIuIBow62FLMr6YR8DiqGiHWSkhf92Zm7vvlTNUGl2bqYT1QzOE9AQgBHHvJuOz9qUmlzPOy8V7cd3AIbrsVX3r6smH7NxJZ9tJ7ZWSxEMaDvsKs271B40oXe3u3yiHPqoqcq2ocBAJUZ/1rNtsBSa3XIP1CiFn19hyAfvX2EICposdNq9u2QEQPEtFRIjoaiURqPAwlc2/Hkowk4HXAYbU0de7ivn4fHDZLwxUz4VgSPqetbgMODo50w24lPG2gJPLJyXnsDXqxy+9Cp8uON944iEeev4JYsvUmP4VjKfjddrjs+iW2sjQz0ddhaOnC7bBiqMu9YWjMmVAM/Z1OXYqcwS43LKQtuJvNdkCi+1sSijFE1cVaIcSnhBCHhBCHgsFgza+vWA+0Z0kGUH6hbhrpauqVh71Ji6rhmD4NdiXcDiv2D/kNa2bK5PI4cnERLxvvKWy7/7ZRJNI5/EsLLqyGosbNppU2BEbW2yV7g94Nvu56F1MB5Xd6wO/WVJYxm+2ApNbgHpLlFvX/sLp9BsBI0eOG1W11Q7EeaN/M/c/uvQZf/b3bm30YODDUiRdnGruoGqmD9cBmDo8FcHx6eYMao1aOTy9jNZ3Dy8Z7C9tuGPbjuoFOfOnpy2UNsJqF3u7UYmTmXo/grrhDKgZi+bzAuXBtnjKb0ap1N5vtgKTW4P4IgAfU2w8A+HbR9neqqpnbAawUlW/qQrs6QhbTqAEd23FgyI9YKltyXma9CNfBemAzh/cEkMkJPDelf9HzyfNKvf2OveuZOxHh/ttGcWo2ashrGIne7tRiDo52o6/DiTsneis/uErGg16spnMIRVOYWkogmckbYm8wEtCauZtvMRXQJoX8MoCnAFxNRNNE9NsAPgLgNUR0DsCr1Z8B4FEAFwCcB/BpAH9Ql6NWSaSzSKRzbR/cW4FmLKqG62A9sJlDuwMggiFWBE9OLuC6gU50b8rw7rtpEB5Hay2sCiEQiaUQNGixepffhSN/8erC74mRjBdNZTozV/30pXKMBjyIxFJYS5e/ajOj7YBEi1rm7UKIASGEXQgxLIR4SAixIIR4lRBinxDi1UKIRfWxQgjxbiHEuBDigBDiaD0Pfj4mu1PNdTnVDK7q74DDZsGLDQruqynlxFwvpYzE77Hj6v4O3XX3ZCaHZy8vbai3Szpcdtx30yD+9fgVQ2WXelhOZJDO5et+8jSCYgOx9dF6RmTulYelm9F2QNLWiv1IGw7GblXsVguu3dXRsMy93hr3Yg7vCeDZl5aQzeVr3sezLy0hnc2XLUvcf3g3kpk8/uXndV1i0oz8fPvrfPI0gr4OJ7wOKy5EVnE2FMdwtxs+p34FlRbrXzPaDkjaOrjL7tR2tPttRfYP+fHCzEpDFgbD0fpZD2zm1rEAEukcXrwSrXkfT07Ow2oh3LonUPL+A8N+HBjyt8zCaj2tHYyGiAoeM7UO6CiFFq27GW0HJG0d3EcDHvzeL+zFUFdzfFnMxoEhP6LJbEMmGOmd7VkNh9WArKc089PzC7hx2L9tRnn/baM4E4rh2GXjzcqqJWxQd2qjGA/6cGYuhguRVcOCe4/XAbfdisuL5RUzZrQdkLT1O7p2oBPvf921Wxa4mNqoZVE1mcnhsZMhzK5ot1cF1oN7I666+jtdGA14al5UjSYzOD69XFEp8sYbB+Fz2vDFFlhYbeTJ0wj29noRjqWQzuV1uUEWQ0TKsOztMncT2g5I2jq4M8ZyVX8HHNbKM1WFEHj2pUX8+bdO4PCHf4Df/cJRvPWTT+HKsvYAH4ml4LBa0GWAL7gWDu8J4JlLizWVTJ65uIi8AO4osZhajNdpw303DeK7x2erHs5sNKFofbt/jUYuqgIwLHMH1t0hS2FW2wEJB3emgMNmwdW7OsraEEwvJfCJx8/hno/+H/yHTz6Fbx2bwauu7cffvvUGRNcy+I3PPF1YB6mE1Lg3SuN/eCyApUSmMFGoGp6cXIDTZinMZt2O+28bRSqbxzeOTddymIYRidVnCEq9kHJIC603TBmB1LqXOqmb1XZA0h6ndaZh7B/y49ETsxBCgIiwmsri0ROz+OaxGTx1QWniuX1vAH/winG87sBAoQa9p9eLdzx0BO946Ai+8ru3V/QFicTq351ajFwIPXJpsWqZ3U/Pz+PQWLcmj5brB/24caQLXzpyGb9551jTGtTCMeOsBxrB7h4PiIDdPV5DvHAkowEPEukcFlfT6NlUAjSr7YCEM3dmAweG/FhZy+Cbx2bwR199Doc+9AP86dePY3ZlDX/0mqvwkz97Jb7y4B34lUMjGxYXD40F8Kl33oLJcBzv+vwRrFaYUGRk96QWxno86PU58UyVdfeFeAqn52IbLAcq8euHR3E+HMczBnvJV0MjGsSMxGW3Yjzow/WDxgbake7yckiz2g5IOLgzG5AzVf/4fz+Px06F8KaDQ/jGf7wDP/qTV+A/vWrfttbEd+8L4uNvP4jj0yv43S8c3dbPJRxLNrRsQES4bU+g6oD7swvKyaBSvb2YN9w4gA6nDV96ujmDPIQQDT95GsHn3nUr/vKN1xu6z9GebYK7SW0HJBzcmQ1cP9iJP3ntVfjE2w/imb94Nf76LQdwy+6A5vLCvft34e9+5QY8ObmA93zpGDIlGofS2TyWEhkEfY3NLG8d68bM8tq2HYub+enkPHxOG26oou3e47DhzTcP4dEX5rC0mq7lUHURS2Wxlql/96/RjAQ8W0onuvfZLbtUNy72m9l2QMLBndmAxUJ4zz378Ms3DtZc+3zzwWF88E378YNTYfzR157fMsdSLro2OvjcWoPe/anJBdy2JwBblTro+28bRbpJC6tS497MGQGtgtthRa/PicsLG0/oZrYdkHBwZ+rCO27fjfe97hr86/NX8BffOrFBrdBI64FirtnViQ6XDUcuaivNXFlew8X51apKMsWvdfOosrDa6I5VoyYwmYXREu6QUinDmTvD1MDv/8I43vPKCXzlmSl86LunCkGukdYDxVgthEO7u3Hk4oKmxz+pjtSr1eb2/tt240JktVC3bxSF8YVttKBaT0YCHkxtKsWdmo2a1nZAwsGdqSt//Nqr8K6XjeGhJy7ifz5+DkBzuydv3RPAZGQVCxr0+E9OziPgddTsLf6GGwbQ6bLhS0ca27EakifPNqu514vRgAdXltc2rP+Y2XZAYt53xrQERIT/+obr8NZbhvEPPziHz/zkAsKxFIgU749Gc3hM1t23L80IIfDU5ALu2NsDS40jEF12K95y8zD+/YVZTScTowhHU3DZLegwwFnRDIwEPMgLYHY5WdhmZtsBCQd3pu5YLISPvOUAXn9gFz703VP41s+n0eN1VL1IaQQHhv1w2iwVF1UvLSQwu5LEyyaqr7cX8+u3jSKTE/j6s41bWJXj9VphwlcrsFnrbnbbAQkHd6Yh2KwW/MOvHcQvXBXE1OIagk2qBzttVtw00lXRROyn5+cBoKrmpVLs6+/ArWPd+PKRyw2bT9tu3an1ZrPW3ey2AxIO7kzDcNgs+F+/cQtecXUQt5XxRW8Eh/cE8OKVFcS36aJ9anIBA34XxnrKN21p5f7bRnFpIVFYoK037dadWm92dbpgt1JhUdXstgMSDu5MQ3E7rPj8bx7GBwzuRKyGW8cCyAvg2Eul6+75vMBTFxZwx3iPIaWN1+0fQI/Xgd/5wjN43zeO46SOoSFaCEcb69vT6lgthKGudTmk2W0HJBzcmR3Hzbu7YbVQ2br76bkYFlfTuFNnSUbislvxtd+/A28+OIR/eW4Gr//4T/Cr/+spfOf4lZIdvHpIpLOIp7LcwLSJkYAH0zK4m9x2QMLBndlx+Jw2XD/YiafL1N2fnFTq7bU0L5VjPOjDX7/lBjz9/lfj//mlazEXTeI9X/o57vofP8THHz9XaDzSS7tNYGoUI+rQjp1gOyDh4M7sSG4dC+C5qWWkslvNzZ6aXMCeXi8G6zC+0e+x43fu3osf/ckr8Nl3HcI1uzrx94+dxZ0f+SHe+5Wf49mXlnR1tLbbBKZGMRrwYCmRwfNTy6a3HZCwEJbZkRzeE8BDT1zEiekVHBpbX9zN5vJ4+uIi7rtpsK6vb7UQ7rmmH/dc048LkTj++Wcv4etHp/Ht567gwJAf77xjd03+Pu00GLuRyGHZj50MATC37YBEV+ZORJeI6AQRPUdER9VtASJ6jIjOqf9XHl/DMA3m1rH14R3FHJ9RVDR6JZDVsDfow3/75evx1J+/Ch98034kMzn86deP4/5P/6xq+WSIyzIlkVr37704Z3rbAYkRZZlXCiFuEkIcUn9+H4DHhRD7ADyu/swwLUXA68BEn2+L3v2pyfVpU43G57ThHbfvxvf/88vxwfuux7HLy1W7SoZjyYbOpm0XZOZ+aSFhetsBST3e4X0AHlZvPwzgTXV4DYbRza1jATx7aWmDJfGTk/O4dqDTcF/xaiAi/Mbtu3FwtAt/870z2+rxNxNRZZDcnboRv8eODpdShd4JShlAf3AXAL5PRM8S0YPqtn4hxKx6ew5Af6knEtGDRHSUiI5GIhGdh8Ew1XPbngBiqSxOzym682Qmh6OXlvAyA1UytSI9eSKxFP7pR+c1Py/cZoOxG4nM3nfCYiqgP7jfJYS4GcDrALybiF5efKdQlv1LFg2FEJ8SQhwSQhwKBoM6D4NhqqcwNFstzRy7vIRUNt8SwR0ADo52480Hh/CZJy5iqsSYuFKEomw9UA4Z3M1uOyDRFdyFEDPq/2EA3wJwGECIiAYAQP0/rPcgGaYeDHW5MdTlLjQzPTW5AKuFcLiJ1gib+bN7r4aVCH/9b6c0PZ6tB8ojg7vZbQckNQd3IvISUYe8DeC1AF4A8AiAB9SHPQDg23oPkmHqxeE9ARy5qGjLf3p+HjcM+9Hhap3FyAG/G7//C+N49MQcfnZhe2+aZCaHlbUM+rksU5LfuH03/uatN5jedkCiJ3PvB/AEET0P4AiA7woh/h3ARwC8hojOAXi1+jPDtCS3jgUwH0/hxStRPD+90jIlmWIefPleDPpd+OB3Tm6ZR1sMT2DanpGAB796aKTZh9Ewam5iEkJcAHBjie0LAF6l56AYplEc3qO0YfzjD88jlxeG+ckYidthxftefy3+05d/jq8/O4Vfu3W05ONkd2qQM3cGbD/A7HDGgz4EvA78+4tzcNgsuHl3a/bc/fINA7hldzf+9ntnEEtmSj5mfTYtB3eGgzuzwyEi3DqmBPRbRrurbvdvFFIaOR9P4x/LSCPDXJZhiuDgzux4pBXBnTpH6tWbG0e68B9uHsbnnriElxZW2MHlcgAABNlJREFUt9wfjiVhtVBTZtMyrQcHd2bHc881fejrcOK11+9q9qFU5M/uvRo2K+G/P7pVGhmOphD0OWse6M2YCw7uzI5nb9CHI3/xalzV3/qdi/2dLvzBK8bxvRdDBd95CXenMsVwcGeYNuN37t6LoS43/upfN0ojuTuVKYaDO8O0GS67Fe9//TU4PRfDV5+ZKmyPxFLo4/F6jAoHd4ZpQ37pwAAOjwXw0e+fQTSZQSaXx8JqmjN3pgAHd4ZpQ4gI/+8brsNiIo1//OF5zMdZBslshMfsMUybcmDYj7fePIzP/fQirh9UzLA4c2cknLkzTBvzp/deDYfVgr/615MAeDA2sw4Hd4ZpY/o6XHj3PRNYWE0DUKSSDANwcGeYtue37tyDkYAbRODuVKYA19wZps1x2a34h187iCMXF2HbAYOfGW1wcGcYE3DL7m7c0qKOlkxz4NM8wzCMCeHgzjAMY0I4uDMMw5gQDu4MwzAmhIM7wzCMCeHgzjAMY0I4uDMMw5gQDu4MwzAmhIQQlR9V74MgigB4qcan9wKYr/goc8HveWfA73lnoOc97xZCBEvd0RLBXQ9EdFQIcajZx9FI+D3vDPg97wzq9Z65LMMwDGNCOLgzDMOYEDME9081+wCaAL/nnQG/551BXd5z29fcGYZhmK2YIXNnGIZhNsHBnWEYxoS0dXAnonuJ6AwRnSei9zX7eBoBEV0iohNE9BwRHW328dQDIvosEYWJ6IWibQEieoyIzqn/m2oyRZn3/AEimlG/6+eI6PXNPEYjIaIRIvoREZ0koheJ6L3qdtN+z9u857p8z21bcyciK4CzAF4DYBrAMwDeLoQ42dQDqzNEdAnAISGEaRs9iOjlAOIAviCE2K9u+xsAi0KIj6gn8m4hxH9p5nEaSZn3/AEAcSHE3zXz2OoBEQ0AGBBCHCOiDgDPAngTgHfBpN/zNu/5V1GH77mdM/fDAM4LIS4IIdIAvgLgviYfE2MAQogfA1jctPk+AA+rtx+G8kdhGsq8Z9MihJgVQhxTb8cAnAIwBBN/z9u857rQzsF9CMBU0c/TqOMH1UIIAN8nomeJ6MFmH0wD6RdCzKq35wD0N/NgGsh7iOi4WrYxTYmiGCIaA3AQwNPYId/zpvcM1OF7bufgvlO5SwhxM4DXAXi3ejm/oxBKLbE964nV8UkA4wBuAjAL4KPNPRzjISIfgG8A+EMhRLT4PrN+zyXec12+53YO7jMARop+Hla3mRohxIz6fxjAt6CUp3YCIbVmKWuX4SYfT90RQoSEEDkhRB7Ap2Gy75qI7FCC3BeFEN9UN5v6ey71nuv1PbdzcH8GwD4i2kNEDgBvA/BIk4+prhCRV12IARF5AbwWwAvbP8s0PALgAfX2AwC+3cRjaQgyyKm8GSb6romIADwE4JQQ4u+L7jLt91zuPdfre25btQwAqJKhfwBgBfBZIcSHm3xIdYWI9kLJ1gHABuBLZnzPRPRlAK+AYoUaAvDfAPwLgK8BGIViD/2rQgjTLECWec+vgHKpLgBcAvB7RfXotoaI7gLwEwAnAOTVzX8OpQZtyu95m/f8dtThe27r4M4wDMOUpp3LMgzDMEwZOLgzDMOYEA7uDMMwJoSDO8MwjAnh4M4wDGNCOLgzDMOYEA7uDMMwJuT/AlqRdH0qqFkwAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>Calculating the Tf-IDF!</h1>"
      ],
      "metadata": {
        "id": "aEF8rVOtwR-B"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import List\n",
        "from copy import copy\n",
        "from collections import Counter, OrderedDict\n",
        "\n",
        "class Corpus:\n",
        "\n",
        "  def __init__(self, vocabulary) -> None:\n",
        "      self._vocabulary = vocabulary\n",
        "      self._lexicon = self._get_lexicon()\n",
        "      self._document_matrix = np.zeros((self.documents_len, self.lexicon_len))\n",
        "      self._tf_idf_matrix = copy(self._document_matrix)\n",
        "      self._vectorize_document()\n",
        "      self._transform_documents()\n",
        "\n",
        "  def _vectorize_document(self):\n",
        "    for doc_i, doc in enumerate(self._vocabulary):\n",
        "      for term in doc:\n",
        "        feture_i = self._lexicon.index(term)\n",
        "        self._document_matrix[doc_i][feture_i] += 1\n",
        "\n",
        "  def _transform_documents(self):\n",
        "    for doc_i in range(len(self._tf_idf_matrix)):\n",
        "      for term_i in range(len(self._tf_idf_matrix[doc_i])):\n",
        "        tf = self.get_tf(doc_i, term_i)\n",
        "        idf = self.get_idf(term_i)\n",
        "        self._tf_idf_matrix[doc_i][term_i] = tf * idf\n",
        "\n",
        "  def _get_lexicon(self):\n",
        "    lexicon = set()\n",
        "    for doc in self._vocabulary:\n",
        "      for token in doc:\n",
        "        lexicon.add(token)\n",
        "    lexicon = list(lexicon)\n",
        "    return sorted(lexicon)\n",
        "\n",
        "  def get_tf(self, doc_index, term_index):\n",
        "    tf = self._document_matrix[doc_index][term_index]\n",
        "    term_document_count = np.sum(self._document_matrix[doc_index])\n",
        "    tf = tf / term_document_count\n",
        "    return tf\n",
        "\n",
        "  def get_idf(self, term_index):\n",
        "    nuber_of_documents_with_t = self.term_document_count(term_index)\n",
        "    idf = self.documents_len / nuber_of_documents_with_t\n",
        "    idf = np.log(idf)\n",
        "    return idf\n",
        "\n",
        "  def term_document_count(self, term_index):\n",
        "    term_column = self._document_matrix[:, term_index]\n",
        "    term_column = [int(x > 0) for x in term_column]\n",
        "    return np.sum(term_column)\n",
        "\n",
        "  def get_term_index(self, term):\n",
        "    try:\n",
        "      index = self._lexicon.index(term)\n",
        "      return index\n",
        "    except ValueError:\n",
        "      return -1\n",
        "\n",
        "  def get_index_term(self, index):\n",
        "    return self._lexicon[index]\n",
        "\n",
        "  @property\n",
        "  def documents_len(self):\n",
        "    return len(self._vocabulary)\n",
        "\n",
        "  @property\n",
        "  def lexicon_len(self):\n",
        "    return len(self._lexicon)\n",
        "\n",
        "  def get_feature_names_out(self):\n",
        "    return self._lexicon\n",
        "\n",
        "  def vector_to_dict(self, vec):\n",
        "    if len(vec) != len(self._tf_idf_matrix[0]):\n",
        "      raise ValueError('Input vector does not belong to this class.')\n",
        "    return OrderedDict((self._lexicon[i], x) for i, x in enumerate(vec))\n",
        "\n",
        "  @property\n",
        "  def tfidf_matrix(self):\n",
        "    return self._tf_idf_matrix\n",
        "\n",
        "  def __len__(self):\n",
        "    return self.documents_len\n",
        "\n",
        "  def __iter__(self):\n",
        "    return self.tfidf_matrix.__iter__()"
      ],
      "metadata": {
        "id": "qyLv3YTOM3mw"
      },
      "execution_count": 91,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "l = [doc['tokenized_text'] for doc in dataset]\n",
        "\n",
        "a = Corpus(l)"
      ],
      "metadata": {
        "id": "cSCLy0gClirv"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from spacy.tokens.token import Token\n",
        "\n",
        "def is_stop_word(token: Token):\n",
        "  return token.is_stop\n",
        "\n",
        "def is_punc_or_sym(token: Token):\n",
        "  part_of_speach = token.pos_\n",
        "  return (part_of_speach == 'PUNCT' or \n",
        "          part_of_speach == 'SYM')\n",
        "  \n",
        "def is_punc_or_sym_or_stop(token: Token):\n",
        "  return is_stop_word(token) or is_punc_or_sym(token)\n",
        "\n",
        "def tokenize_text(input_string: str):\n",
        "  spacy_doc = nlp(input_string)\n",
        "  transformed = [token.lemma_ for token in spacy_doc \n",
        "                 if not is_punc_or_sym_or_stop(token)]\n",
        "  transformed = [lemma.lower() for lemma in transformed]\n",
        "  return transformed"
      ],
      "metadata": {
        "id": "DV_ridC2RW8T"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import math\n",
        "\n",
        "\"\"\"\n",
        "Author: Natural Language Processing in Action: Understanding, analyzing, \n",
        "  and generating text with Python | Hobson Lane,Cole Howard,Hannes Hapke\n",
        "\"\"\"\n",
        "\n",
        "def cosine_sim(vec1, vec2):\n",
        "     \"\"\" Let's convert our dictionaries to lists for easier matching.\"\"\"\n",
        "    #  vec1 = [val for val in vec1.values()]\n",
        "    #  vec2 = [val for val in vec2.values()]\n",
        "\n",
        "     dot_prod = 0\n",
        "     for i, v in enumerate(vec1):\n",
        "         dot_prod += v * vec2[i]\n",
        "\n",
        "     mag_1 = math.sqrt(sum([x**2 for x in vec1]))\n",
        "     mag_2 = math.sqrt(sum([x**2 for x in vec2]))\n",
        "\n",
        "     return dot_prod / (mag_1 * mag_2)"
      ],
      "metadata": {
        "id": "-5brpMuQxvdQ"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class Query:\n",
        "\n",
        "  def __init__(self, query, corpus: Corpus):\n",
        "    self._query = tokenize_text(query)\n",
        "    self._corpus = corpus\n",
        "    self._tf_idf = self._tf_idf()\n",
        "\n",
        "  def _vectorize_query(self):\n",
        "    vec = np.zeros(self._corpus.lexicon_len)\n",
        "    for token in self._query:\n",
        "      token_index = self._corpus.get_term_index(token)\n",
        "      if token_index < 0:\n",
        "        continue  # out of vocabulary\n",
        "      vec[token_index] += 1\n",
        "    return vec\n",
        "\n",
        "  def _tf_idf(self):\n",
        "    count_vector = self._vectorize_query()\n",
        "\n",
        "    tf_vector = copy(count_vector)\n",
        "    for i in range(len(count_vector)):\n",
        "      tf = count_vector[i]\n",
        "      term_document_count = np.sum(count_vector)\n",
        "      tf = tf / term_document_count\n",
        "      tf_vector[i] = tf\n",
        "\n",
        "    idf_vector = copy(count_vector)\n",
        "    for i in range(len(count_vector)):\n",
        "      nuber_of_documents_with_t = self._corpus.term_document_count(i)\n",
        "      idf = self._corpus.documents_len / nuber_of_documents_with_t\n",
        "      idf = np.log(idf)\n",
        "      idf_vector[i] = idf\n",
        "\n",
        "    return tf_vector * idf_vector\n",
        "\n",
        "  @property\n",
        "  def tf_idf(self):\n",
        "    return self._tf_idf\n",
        "\n",
        "  @property\n",
        "  def dict_tf_idf(self):\n",
        "    dict_tf_idf = self._corpus.vector_to_dict(self.tf_idf)\n",
        "    return {k: v for k, v in dict_tf_idf.items() if v > 0}\n",
        "\n",
        "  def search_result(self):\n",
        "    result = list()\n",
        "    for i, doc in enumerate(self._corpus):\n",
        "      score = cosine_sim(doc, self.tf_idf)\n",
        "      title = dataset[i]['title']\n",
        "      link = dataset[i]['url']\n",
        "      result.append(dict(title=title, score=score, link=link))\n",
        "    return sorted(result, key=lambda x: x['score'], reverse=True)"
      ],
      "metadata": {
        "id": "qifhY--6Q61M"
      },
      "execution_count": 118,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query = Query('When did the spanish flu happen', a)\n",
        "query.search_result()[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PtUB6AJpSh3L",
        "outputId": "d98cbc0e-81ec-41e2-d67f-934bbead66c1"
      },
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'link': 'https://en.wikipedia.org/wiki/Spanish_flu',\n",
              "  'score': 0.32722431725410606,\n",
              "  'title': 'Spanish flu'},\n",
              " {'link': 'https://en.wikipedia.org/wiki/Swine_influenza',\n",
              "  'score': 0.27806276291630766,\n",
              "  'title': 'Swine influenza'},\n",
              " {'link': 'https://en.wikipedia.org/wiki/Pandemic',\n",
              "  'score': 0.1262668890689719,\n",
              "  'title': 'Pandemic'},\n",
              " {'link': 'https://en.wikipedia.org/wiki/Unified_Victim_Identification_System',\n",
              "  'score': 0.024451792853454935,\n",
              "  'title': 'Unified Victim Identification System'},\n",
              " {'link': 'https://en.wikipedia.org/wiki/Epidemiology_of_HIV/AIDS',\n",
              "  'score': 0.0,\n",
              "  'title': 'Epidemiology of HIV/AIDS'}]"
            ]
          },
          "metadata": {},
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>Adding the TF-IDF vectors to our dataset!</h1>"
      ],
      "metadata": {
        "id": "VsBV8pWNww6F"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_dataset = copy(dataset)\n",
        "\n",
        "for i, doc in enumerate(a):\n",
        "  new_dataset[i].update({'tf_idf': doc.tolist()})"
      ],
      "metadata": {
        "id": "HUMpvOmLwulc"
      },
      "execution_count": 122,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "{k:v for k, v in a.vector_to_dict(new_dataset[0]['tf_idf']).items() if v > 0}"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MpL-FLoCxHBs",
        "outputId": "f123a8f5-f7f8-404f-828f-fb5a8e25f205"
      },
      "execution_count": 128,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'\\n': 0.004828218113241659,\n",
              " '14th': 0.037023824295698664,\n",
              " '1918': 0.029147151789335645,\n",
              " '75–200': 0.037023824295698664,\n",
              " 'affect': 0.014911208965524645,\n",
              " 'aids': 0.016662921236288944,\n",
              " 'black': 0.037023824295698664,\n",
              " 'century': 0.01873475710894752,\n",
              " 'continent': 0.037023824295698664,\n",
              " 'cov-2': 0.024539593742651956,\n",
              " 'covid-19': 0.016662921236288944,\n",
              " 'current': 0.029147151789335645,\n",
              " 'death': 0.00977501437753536,\n",
              " 'demo': 0.037023824295698664,\n",
              " 'disease': 0.0220685370809864,\n",
              " 'endemic': 0.04907918748530391,\n",
              " 'epidemic': 0.010858084602584504,\n",
              " 'estimate': 0.010858084602584504,\n",
              " 'exclude': 0.037023824295698664,\n",
              " 'fatal': 0.037023824295698664,\n",
              " 'flu': 0.02127047928297263,\n",
              " 'generally': 0.024539593742651956,\n",
              " 'globe': 0.037023824295698664,\n",
              " 'greek': 0.037023824295698664,\n",
              " 'history': 0.04254095856594526,\n",
              " 'hiv': 0.016662921236288944,\n",
              " 'human': 0.00977501437753536,\n",
              " 'include': 0.01575334501272603,\n",
              " 'infected': 0.03332584247257789,\n",
              " 'infectious': 0.016662921236288944,\n",
              " 'influenza': 0.021716169205169007,\n",
              " 'instance': 0.037023824295698664,\n",
              " 'kill': 0.02127047928297263,\n",
              " 'know': 0.010858084602584504,\n",
              " 'large': 0.02982241793104929,\n",
              " 'later': 0.024539593742651956,\n",
              " 'million': 0.010858084602584504,\n",
              " 'multiple': 0.024539593742651956,\n",
              " 'number': 0.08508191713189052,\n",
              " 'occur': 0.02127047928297263,\n",
              " 'pan': 0.037023824295698664,\n",
              " 'pandemic': 0.03379752679269161,\n",
              " 'people': 0.04887507188767679,\n",
              " 'plague': 0.024539593742651956,\n",
              " 'recorded': 0.037023824295698664,\n",
              " 'recurrence': 0.037023824295698664,\n",
              " 'region': 0.05829430357867129,\n",
              " 'sars': 0.02127047928297263,\n",
              " 'seasonal': 0.037023824295698664,\n",
              " 'simultaneously': 0.037023824295698664,\n",
              " 'smallpox': 0.024539593742651956,\n",
              " 'spanish': 0.024539593742651956,\n",
              " 'spread': 0.01955002875507072,\n",
              " 'stable': 0.05829430357867129,\n",
              " 'substantial': 0.037023824295698664,\n",
              " 'term': 0.024539593742651956,\n",
              " 'tuberculosis': 0.024539593742651956,\n",
              " 'widespread': 0.05829430357867129,\n",
              " 'worldwide': 0.03746951421789504,\n",
              " 'δῆμος': 0.037023824295698664,\n",
              " 'πᾶν': 0.037023824295698664}"
            ]
          },
          "metadata": {},
          "execution_count": 128
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h1>Saving the transfomred dataset</h1>"
      ],
      "metadata": {
        "id": "AveB_QyA2AbY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "\n",
        "json_datset = json.dumps(new_dataset)\n",
        "\n",
        "with open('tf_idf_dataset.json', 'w') as f:\n",
        "  f.write(json_datset)"
      ],
      "metadata": {
        "id": "PIC1OMj52Gsk"
      },
      "execution_count": 129,
      "outputs": []
    }
  ]
}