Toxic_Speech_Classifier.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "JgcMidzvYYl_"
      },
      "source": [
        "# [ DATA COLLECTION AND PROCESSING ]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "G5uZVLAKYFSo"
      },
      "source": [
        "# 1 | Preparing the data frame."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xNZvL9SZYFSr"
      },
      "source": [
        "## 1.1 | Hate Speech Filipino Dataset (source 1)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "QJjAvLliYFSr"
      },
      "source": [
        "### Load the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "SI4tZ71kYFSs"
      },
      "outputs": [],
      "source": [
        "# Hate Speech in Filipino: Contains 10k tweets (training set) that are labeled as hate speech\n",
        "# or non-hate speech. It also comes with 4,232 validation and 10k testing samples.\n",
        "# Collected during the 2016 Philippine Presidential Elections.\n",
        "#\n",
        "# Load the datasets from the local csv files.\n",
        "# Datasets are from Hugging Face (ref: https://huggingface.co/datasets/hate_speech_filipino)\n",
        "#\n",
        "# Dataset Columns\n",
        "# - text (String) : The input text (i.e., tweet).\n",
        "# - label (int) : 0 means 'non-hate speech' while 1 means 'hate speech.'\n",
        "\n",
        "import pandas as pd\n",
        "\n",
        "local_sub_path = 'datasets/Hate Speech Filipino/'\n",
        "\n",
        "hate_speech_fil_train_df = pd.read_csv(local_sub_path + 'hate_speech_filipino_train.csv')\n",
        "hate_speech_fil_test_df = pd.read_csv(local_sub_path + 'hate_speech_filipino_train.csv')\n",
        "hate_speech_fil_validate_df = pd.read_csv(local_sub_path + 'hate_speech_filipino_validate.csv')\n",
        "\n",
        "# Combine all the datasets into a single data frame.\n",
        "hate_speech_fil_df = pd.concat([hate_speech_fil_test_df, hate_speech_fil_train_df, hate_speech_fil_validate_df])\n",
        "\n",
        "# Reset the index of hate_speech_fil_df.\n",
        "hate_speech_fil_df.reset_index(drop=True, inplace=True)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zVNp1lUfYFSt"
      },
      "source": [
        "### Inspect the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "WLXgy5qhYFSt",
        "outputId": "b36e7760-4151-408c-80fd-f13d29ef9b41"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>Mar Roxas on the rise, w/ momentum, machinery,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>@chelseapailmao</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>Alan Cayetano 'confirms' Palace, Roxas, Poe be...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>Mas nakakainis ad ni Mar kaysa kay Binay.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>Preliminary and partial results coming in sugg...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  label\n",
              "0     Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1     Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2     Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3            @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4     Binay with selective amnesia, forgetting about...      0\n",
              "...                                                 ...    ...\n",
              "9995  Mar Roxas on the rise, w/ momentum, machinery,...      0\n",
              "9996                                    @chelseapailmao      1\n",
              "9997  Alan Cayetano 'confirms' Palace, Roxas, Poe be...      0\n",
              "9998          Mas nakakainis ad ni Mar kaysa kay Binay.      1\n",
              "9999  Preliminary and partial results coming in sugg...      0\n",
              "\n",
              "[10000 rows x 2 columns]"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hate_speech_fil_train_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "xn70ZixAYFSu",
        "outputId": "6633a817-3e90-4724-e1e2-973665d11205"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9995</th>\n",
              "      <td>Mar Roxas on the rise, w/ momentum, machinery,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9996</th>\n",
              "      <td>@chelseapailmao</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9997</th>\n",
              "      <td>Alan Cayetano 'confirms' Palace, Roxas, Poe be...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9998</th>\n",
              "      <td>Mas nakakainis ad ni Mar kaysa kay Binay.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9999</th>\n",
              "      <td>Preliminary and partial results coming in sugg...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10000 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  label\n",
              "0     Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1     Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2     Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3            @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4     Binay with selective amnesia, forgetting about...      0\n",
              "...                                                 ...    ...\n",
              "9995  Mar Roxas on the rise, w/ momentum, machinery,...      0\n",
              "9996                                    @chelseapailmao      1\n",
              "9997  Alan Cayetano 'confirms' Palace, Roxas, Poe be...      0\n",
              "9998          Mas nakakainis ad ni Mar kaysa kay Binay.      1\n",
              "9999  Preliminary and partial results coming in sugg...      0\n",
              "\n",
              "[10000 rows x 2 columns]"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hate_speech_fil_test_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "M_OiRSiUYFSu",
        "outputId": "88837994-d2eb-4405-f28a-3440ebb46440"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Escudero denies betraying Poe after meeting wi...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Hndi ko makita yung sa one more chance saka ka...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Mar Roxas is now addressing the crowd gathered...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@ImYourBaeMax perfect! Para makaharap ni Duter...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>#OnlyBinayPriority4Ps Wag nating hayaan na mal...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4227</th>\n",
              "      <td>Walang katumbas na halaga ng pera ang BOTO at ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4228</th>\n",
              "      <td>Mar Roxas says presidential race is now only b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4229</th>\n",
              "      <td>Ms Miriam to withdraw presidency to give away ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4230</th>\n",
              "      <td>@B_RA_Ve I feel legitimately sick. Apparently,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4231</th>\n",
              "      <td>@TheJhonitor absolutely, but don't be so sure ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4232 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   text  label\n",
              "0     Escudero denies betraying Poe after meeting wi...      0\n",
              "1     Hndi ko makita yung sa one more chance saka ka...      1\n",
              "2     Mar Roxas is now addressing the crowd gathered...      0\n",
              "3     @ImYourBaeMax perfect! Para makaharap ni Duter...      0\n",
              "4     #OnlyBinayPriority4Ps Wag nating hayaan na mal...      0\n",
              "...                                                 ...    ...\n",
              "4227  Walang katumbas na halaga ng pera ang BOTO at ...      0\n",
              "4228  Mar Roxas says presidential race is now only b...      0\n",
              "4229  Ms Miriam to withdraw presidency to give away ...      0\n",
              "4230  @B_RA_Ve I feel legitimately sick. Apparently,...      0\n",
              "4231  @TheJhonitor absolutely, but don't be so sure ...      0\n",
              "\n",
              "[4232 rows x 2 columns]"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hate_speech_fil_validate_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "cHHNq12-YFSu",
        "outputId": "8d01ac0c-8193-482a-e10a-86a96fd303e8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24227</th>\n",
              "      <td>Walang katumbas na halaga ng pera ang BOTO at ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24228</th>\n",
              "      <td>Mar Roxas says presidential race is now only b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24229</th>\n",
              "      <td>Ms Miriam to withdraw presidency to give away ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24230</th>\n",
              "      <td>@B_RA_Ve I feel legitimately sick. Apparently,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24231</th>\n",
              "      <td>@TheJhonitor absolutely, but don't be so sure ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>24232 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "24227  Walang katumbas na halaga ng pera ang BOTO at ...      0\n",
              "24228  Mar Roxas says presidential race is now only b...      0\n",
              "24229  Ms Miriam to withdraw presidency to give away ...      0\n",
              "24230  @B_RA_Ve I feel legitimately sick. Apparently,...      0\n",
              "24231  @TheJhonitor absolutely, but don't be so sure ...      0\n",
              "\n",
              "[24232 rows x 2 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hate_speech_fil_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "97qK7jjAYFSv",
        "outputId": "ead1e2fe-fe66-4c59-ce57-f1bbcabf5f91"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 24232 entries, 0 to 24231\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype \n",
            "---  ------  --------------  ----- \n",
            " 0   text    24232 non-null  object\n",
            " 1   label   24232 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 378.8+ KB\n"
          ]
        }
      ],
      "source": [
        "# Check the basic information of the data frame.\n",
        "hate_speech_fil_df.info()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "weZlr19ZYFSv"
      },
      "source": [
        "### Clean the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "nTXxasqnYFSv",
        "outputId": "1e7fcb94-080b-46e7-d8b5-605a0e959d98"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "text     0\n",
              "label    0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for null values.\n",
        "hate_speech_fil_df.isnull().sum()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "v5CALYlDaJMV"
      },
      "source": [
        "It is important to remove the duplicate entries, since they might potentially skew the results/model later on."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "_172sgXXYFSv",
        "outputId": "c99bd366-44df-4a40-a287-2475b8e6e110"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "10356"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "hate_speech_fil_df.duplicated().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "YrKFAKKjYFSw",
        "outputId": "58fa243e-19bc-4eba-f2fc-538ba8b45e84"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>452</th>\n",
              "      <td>#NogNogHambog</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>481</th>\n",
              "      <td>#OnlyBinayInMakatiSanKaPa</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>543</th>\n",
              "      <td>#OnlyBinayInMakatiSanKaPa</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>576</th>\n",
              "      <td>#OnlyBinayInMakatiSanKaPa</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>598</th>\n",
              "      <td>#OnlyBinayUNAsaIlocos</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24013</th>\n",
              "      <td>#OnlyBinayUNAsaIlocos</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24029</th>\n",
              "      <td>#onlybinay</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24042</th>\n",
              "      <td>#PHvote</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24158</th>\n",
              "      <td>#PiliPinasDebates2016</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24209</th>\n",
              "      <td>#OnlyBinayNangabakIlocos</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>10356 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                            text  label\n",
              "452                #NogNogHambog      1\n",
              "481    #OnlyBinayInMakatiSanKaPa      0\n",
              "543    #OnlyBinayInMakatiSanKaPa      0\n",
              "576    #OnlyBinayInMakatiSanKaPa      0\n",
              "598        #OnlyBinayUNAsaIlocos      0\n",
              "...                          ...    ...\n",
              "24013      #OnlyBinayUNAsaIlocos      0\n",
              "24029                 #onlybinay      0\n",
              "24042                    #PHvote      0\n",
              "24158      #PiliPinasDebates2016      1\n",
              "24209   #OnlyBinayNangabakIlocos      0\n",
              "\n",
              "[10356 rows x 2 columns]"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the duplicates.\n",
        "hate_speech_fil_df[hate_speech_fil_df.duplicated()]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "cSXz1AjsYFSw"
      },
      "outputs": [],
      "source": [
        "# Drop the duplicates.\n",
        "hate_speech_fil_df.drop_duplicates(inplace=True)\n",
        "\n",
        "# Reset the index.\n",
        "hate_speech_fil_df.reset_index(drop=True, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "23wsd7ZCYFSw",
        "outputId": "78f160e1-e73f-4140-81fa-ae5934168eb7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Verify that there are no duplicates.\n",
        "hate_speech_fil_df.duplicated().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "Sf9sfLowYFSw",
        "outputId": "27fcf782-eb01-42d1-c90f-3b53c663583c"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13871</th>\n",
              "      <td>Walang katumbas na halaga ng pera ang BOTO at ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13872</th>\n",
              "      <td>Mar Roxas says presidential race is now only b...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13873</th>\n",
              "      <td>Ms Miriam to withdraw presidency to give away ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13874</th>\n",
              "      <td>@B_RA_Ve I feel legitimately sick. Apparently,...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13875</th>\n",
              "      <td>@TheJhonitor absolutely, but don't be so sure ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>13876 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "13871  Walang katumbas na halaga ng pera ang BOTO at ...      0\n",
              "13872  Mar Roxas says presidential race is now only b...      0\n",
              "13873  Ms Miriam to withdraw presidency to give away ...      0\n",
              "13874  @B_RA_Ve I feel legitimately sick. Apparently,...      0\n",
              "13875  @TheJhonitor absolutely, but don't be so sure ...      0\n",
              "\n",
              "[13876 rows x 2 columns]"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "hate_speech_fil_df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "kOXwkMKYYFSw"
      },
      "source": [
        "## 1.2 | Toxic Tweets Dataset (source 2)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "yo7choHuYFSw"
      },
      "source": [
        "### Load the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "CjO5QcbDYFSx"
      },
      "outputs": [],
      "source": [
        "# Toxic Tweets Dataset\n",
        "# A balanced dataset of tweets containing hate speech and offensive language.\n",
        "#\n",
        "# Load the dataset from the local csv file.\n",
        "# Dataset is from Kaggle (ref: https://www.kaggle.com/datasets/ashwiniyer176/toxic-tweets-dataset)\n",
        "#\n",
        "# Dataset Columns\n",
        "# - tweet (String) : The input text (i.e., tweet).\n",
        "# - Toxicity (int) : Label indicating whether a tweet is toxic(1) or not(0).\n",
        "\n",
        "local_sub_path = 'datasets/Toxic Tweets/'\n",
        "\n",
        "raw_toxic_tweets_df = pd.read_csv(local_sub_path + 'FinalBalancedDataset.csv')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "_JjsnNWnYFSx"
      },
      "source": [
        "### Inspect the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "R3ePDSVNYFSx",
        "outputId": "8fc67c14-67ae-4253-9ff3-a9bd0126300f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>Toxicity</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>@user when a father is dysfunctional and is s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>bihday your majesty</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>factsguide: society now    #motivation</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56740</th>\n",
              "      <td>56740</td>\n",
              "      <td>1</td>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56741</th>\n",
              "      <td>56741</td>\n",
              "      <td>1</td>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56742</th>\n",
              "      <td>56742</td>\n",
              "      <td>1</td>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56743</th>\n",
              "      <td>56743</td>\n",
              "      <td>1</td>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56744</th>\n",
              "      <td>56744</td>\n",
              "      <td>0</td>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>56745 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Unnamed: 0  Toxicity                                              tweet\n",
              "0               0         0   @user when a father is dysfunctional and is s...\n",
              "1               1         0  @user @user thanks for #lyft credit i can't us...\n",
              "2               2         0                                bihday your majesty\n",
              "3               3         0  #model   i love u take with u all the time in ...\n",
              "4               4         0             factsguide: society now    #motivation\n",
              "...           ...       ...                                                ...\n",
              "56740       56740         1  you's a muthaf***in lie &#8220;@LifeAsKing: @2...\n",
              "56741       56741         1  you've gone and broke the wrong heart baby, an...\n",
              "56742       56742         1  young buck wanna eat!!.. dat nigguh like I ain...\n",
              "56743       56743         1              youu got wild bitches tellin you lies\n",
              "56744       56744         0  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...\n",
              "\n",
              "[56745 rows x 3 columns]"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "raw_toxic_tweets_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "NWdYtRrVYFSx",
        "outputId": "41943c20-3f26-49f8-966f-78f2152047ce"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 56745 entries, 0 to 56744\n",
            "Data columns (total 3 columns):\n",
            " #   Column      Non-Null Count  Dtype \n",
            "---  ------      --------------  ----- \n",
            " 0   Unnamed: 0  56745 non-null  int64 \n",
            " 1   Toxicity    56745 non-null  int64 \n",
            " 2   tweet       56745 non-null  object\n",
            "dtypes: int64(2), object(1)\n",
            "memory usage: 1.3+ MB\n"
          ]
        }
      ],
      "source": [
        "# Check the basic information of the data frame.\n",
        "raw_toxic_tweets_df.info()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xmt6kkJQYFSx"
      },
      "source": [
        "### Clean the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "sePvxHANYFSx",
        "outputId": "7404a123-5afe-4931-a1a5-4fc8980d8345"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Unnamed: 0    0\n",
              "Toxicity      0\n",
              "tweet         0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for null values.\n",
        "raw_toxic_tweets_df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "PjQ_lnEhYFSx",
        "outputId": "1b83cd68-32a8-4455-e383-fe9058105416"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "raw_toxic_tweets_df.duplicated().sum()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "D__XNEEWYFSx"
      },
      "source": [
        "### Restructure the data frame."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "eSPRN1RRYFSx",
        "outputId": "20c64ab9-2d4d-4ab8-aecf-5166f9391201"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>@user when a father is dysfunctional and is s...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bihday your majesty</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>factsguide: society now    #motivation</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56740</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56741</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56742</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56743</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56744</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>56745 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0       @user when a father is dysfunctional and is s...      0\n",
              "1      @user @user thanks for #lyft credit i can't us...      0\n",
              "2                                    bihday your majesty      0\n",
              "3      #model   i love u take with u all the time in ...      0\n",
              "4                 factsguide: society now    #motivation      0\n",
              "...                                                  ...    ...\n",
              "56740  you's a muthaf***in lie &#8220;@LifeAsKing: @2...      1\n",
              "56741  you've gone and broke the wrong heart baby, an...      1\n",
              "56742  young buck wanna eat!!.. dat nigguh like I ain...      1\n",
              "56743              youu got wild bitches tellin you lies      1\n",
              "56744  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...      0\n",
              "\n",
              "[56745 rows x 2 columns]"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# The data frame should have only a text and label features.\n",
        "\n",
        "toxic_tweets_df = raw_toxic_tweets_df.copy()\n",
        "toxic_tweets_df.rename(columns={'tweet': 'text', 'Toxicity': 'label'}, inplace=True)\n",
        "toxic_tweets_df = toxic_tweets_df[['text', 'label']]\n",
        "\n",
        "toxic_tweets_df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5MhyZABhYFSy"
      },
      "source": [
        "### Clean the dataset again."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "YtqXOxZJYFSy",
        "outputId": "55f9a373-a8a2-403b-b4bf-64186c927c8c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "2432"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "toxic_tweets_df.duplicated().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "3MqkOaGrYFSy",
        "outputId": "8a8cffc0-9b15-4acc-8424-ebc37e4dce72"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>96</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>117</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>162</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>218</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31886</th>\n",
              "      <td>100 amazing health benefits of cucumbers!    #...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31904</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31930</th>\n",
              "      <td>@user #feminismiscancer #feminismisterrorism #...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31951</th>\n",
              "      <td>i've just posted a new blog: #secondlife #lone...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31960</th>\n",
              "      <td>@user #sikh #temple vandalised in in #calgary,...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2432 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "46     #model   i love u take with u all the time in ...      0\n",
              "96     #model   i love u take with u all the time in ...      0\n",
              "117    #model   i love u take with u all the time in ...      0\n",
              "162    #model   i love u take with u all the time in ...      0\n",
              "218    #model   i love u take with u all the time in ...      0\n",
              "...                                                  ...    ...\n",
              "31886  100 amazing health benefits of cucumbers!    #...      0\n",
              "31904  #model   i love u take with u all the time in ...      0\n",
              "31930  @user #feminismiscancer #feminismisterrorism #...      1\n",
              "31951  i've just posted a new blog: #secondlife #lone...      0\n",
              "31960  @user #sikh #temple vandalised in in #calgary,...      1\n",
              "\n",
              "[2432 rows x 2 columns]"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the duplicates.\n",
        "toxic_tweets_df[toxic_tweets_df.duplicated()]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "-jr5qVa9YFSy",
        "outputId": "11b7e5c0-c584-44f0-8047-77c5a1285edb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>@user when a father is dysfunctional and is s...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>@user @user thanks for #lyft credit i can't us...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>bihday your majesty</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>#model   i love u take with u all the time in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>factsguide: society now    #motivation</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54308</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54309</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54310</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54311</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>54312</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>54313 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0       @user when a father is dysfunctional and is s...      0\n",
              "1      @user @user thanks for #lyft credit i can't us...      0\n",
              "2                                    bihday your majesty      0\n",
              "3      #model   i love u take with u all the time in ...      0\n",
              "4                 factsguide: society now    #motivation      0\n",
              "...                                                  ...    ...\n",
              "54308  you's a muthaf***in lie &#8220;@LifeAsKing: @2...      1\n",
              "54309  you've gone and broke the wrong heart baby, an...      1\n",
              "54310  young buck wanna eat!!.. dat nigguh like I ain...      1\n",
              "54311              youu got wild bitches tellin you lies      1\n",
              "54312  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...      0\n",
              "\n",
              "[54313 rows x 2 columns]"
            ]
          },
          "execution_count": 21,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Drop the duplicates.\n",
        "toxic_tweets_df.drop_duplicates(inplace=True)\n",
        "\n",
        "# Reset the index.\n",
        "toxic_tweets_df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "toxic_tweets_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "YJbJyDP1YFSy",
        "outputId": "8f0a64f4-080a-4ce6-ca18-5baf0a255b9a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Verify that there are no duplicates.\n",
        "toxic_tweets_df.duplicated().sum()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "LGvkwbmWYFSy"
      },
      "source": [
        "## 1.3 | Hate Speech and Offensive Language Dataset (source 3)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "8-mag_8BYFSy"
      },
      "source": [
        "### Load the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "EuNuhx-WYFSy"
      },
      "outputs": [],
      "source": [
        "# Dataset using Twitter data. The text is classified as: hate-speech, offensive language, and neither.\n",
        "# This dataset contains text that can be considered racist, sexist, homophobic, or generally offensive.\n",
        "#\n",
        "# Load the dataset from the local csv file.\n",
        "# Dataset is from Kaggle (ref: https://www.kaggle.com/datasets/mrmorj/hate-speech-and-offensive-language-dataset)\n",
        "# \n",
        "# Dataset Columns\n",
        "# - counts (int) : Number of CrowdFlower users who coded each tweet (min is 3, sometimes more users coded a tweet when judgments were determined to be unreliable by CF).\n",
        "# - hate_speeches (int) : Number of CF users who judged the tweet to be hate speech.\n",
        "# - offensive_languages (int) : Number of CF users who judged the tweet to be offensive.\n",
        "# - neither (int) : Number of CF users who judged the tweet to be neither offensive nor non-offensive.\n",
        "# - class (int) : Class label for majority of CF users. 0 - hate speech 1 - offensive language 2 - neither.\n",
        "# - tweet (String) : Text tweet.\n",
        "#\n",
        "# Note: For simplicity sake, we will be only using the 'tweet' and 'class' columns. They are more than enough for this context.\n",
        "\n",
        "local_sub_path = 'datasets/Hate Speech and Offensive Language/'\n",
        "\n",
        "raw_hs_and_offensive_lang_df = pd.read_csv(local_sub_path + 'labeled_data.csv')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "YFCwVrIBYFSy"
      },
      "source": [
        "### Inspect the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "wLeb3GhSYFSy",
        "outputId": "d45e8204-cb91-480c-9e94-705671d635c0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>count</th>\n",
              "      <th>hate_speech</th>\n",
              "      <th>offensive_language</th>\n",
              "      <th>neither</th>\n",
              "      <th>class</th>\n",
              "      <th>tweet</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24778</th>\n",
              "      <td>25291</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24779</th>\n",
              "      <td>25292</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24780</th>\n",
              "      <td>25294</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24781</th>\n",
              "      <td>25295</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24782</th>\n",
              "      <td>25296</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>24783 rows × 7 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "       Unnamed: 0  count  hate_speech  offensive_language  neither  class  \\\n",
              "0               0      3            0                   0        3      2   \n",
              "1               1      3            0                   3        0      1   \n",
              "2               2      3            0                   3        0      1   \n",
              "3               3      3            0                   2        1      1   \n",
              "4               4      6            0                   6        0      1   \n",
              "...           ...    ...          ...                 ...      ...    ...   \n",
              "24778       25291      3            0                   2        1      1   \n",
              "24779       25292      3            0                   1        2      2   \n",
              "24780       25294      3            0                   3        0      1   \n",
              "24781       25295      6            0                   6        0      1   \n",
              "24782       25296      3            0                   0        3      2   \n",
              "\n",
              "                                                   tweet  \n",
              "0      !!! RT @mayasolovely: As a woman you shouldn't...  \n",
              "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...  \n",
              "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...  \n",
              "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...  \n",
              "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...  \n",
              "...                                                  ...  \n",
              "24778  you's a muthaf***in lie &#8220;@LifeAsKing: @2...  \n",
              "24779  you've gone and broke the wrong heart baby, an...  \n",
              "24780  young buck wanna eat!!.. dat nigguh like I ain...  \n",
              "24781              youu got wild bitches tellin you lies  \n",
              "24782  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...  \n",
              "\n",
              "[24783 rows x 7 columns]"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "raw_hs_and_offensive_lang_df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "Fh-4Msu3YFSz",
        "outputId": "bc63355f-3e98-495f-c8a0-4b259d07f9a3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 24783 entries, 0 to 24782\n",
            "Data columns (total 7 columns):\n",
            " #   Column              Non-Null Count  Dtype \n",
            "---  ------              --------------  ----- \n",
            " 0   Unnamed: 0          24783 non-null  int64 \n",
            " 1   count               24783 non-null  int64 \n",
            " 2   hate_speech         24783 non-null  int64 \n",
            " 3   offensive_language  24783 non-null  int64 \n",
            " 4   neither             24783 non-null  int64 \n",
            " 5   class               24783 non-null  int64 \n",
            " 6   tweet               24783 non-null  object\n",
            "dtypes: int64(6), object(1)\n",
            "memory usage: 1.3+ MB\n"
          ]
        }
      ],
      "source": [
        "# Check the basic information of the data frame.\n",
        "raw_hs_and_offensive_lang_df.info()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "u7KMemvcYFS2"
      },
      "source": [
        "### Clean the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "R-vAz_y0YFS2",
        "outputId": "e5c83ef0-08bf-429d-f259-e044c430ca34"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Unnamed: 0            0\n",
              "count                 0\n",
              "hate_speech           0\n",
              "offensive_language    0\n",
              "neither               0\n",
              "class                 0\n",
              "tweet                 0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for null values.\n",
        "raw_hs_and_offensive_lang_df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "p7wpMmGAYFS2",
        "outputId": "ac703ccc-c1b3-41d8-c113-2f44640eab59"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "raw_hs_and_offensive_lang_df.duplicated().sum()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "bpgvKqWrYFS2"
      },
      "source": [
        "### Restructure the data frame."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "X-UyelKwYFS3",
        "outputId": "2de4fa87-bd29-466a-cea1-208408f4d100"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24778</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24779</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24780</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24781</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24782</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>24783 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      !!! RT @mayasolovely: As a woman you shouldn't...      0\n",
              "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...      1\n",
              "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...      1\n",
              "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...      1\n",
              "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...      1\n",
              "...                                                  ...    ...\n",
              "24778  you's a muthaf***in lie &#8220;@LifeAsKing: @2...      1\n",
              "24779  you've gone and broke the wrong heart baby, an...      0\n",
              "24780  young buck wanna eat!!.. dat nigguh like I ain...      1\n",
              "24781              youu got wild bitches tellin you lies      1\n",
              "24782  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...      0\n",
              "\n",
              "[24783 rows x 2 columns]"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# The data frame should have only a text and label features.\n",
        "\n",
        "hs_and_offensive_lang_df = raw_hs_and_offensive_lang_df.copy()\n",
        "hs_and_offensive_lang_df = hs_and_offensive_lang_df[['tweet', 'class']]\n",
        "hs_and_offensive_lang_df.rename(columns={'tweet': 'text', 'class': 'label'}, inplace=True)\n",
        "\n",
        "# The original labels are 0, 1, and 2. They each means 'hate speech', 'offensive language', and 'neither' respectively.\n",
        "# The original labels 0 and 1 will be mapped to 1 (means toxic) and 2 will be mapped to 0 (means non-toxic)\n",
        "mapping = {0: 1, 1: 1, 2: 0}\n",
        "hs_and_offensive_lang_df['label'] = hs_and_offensive_lang_df['label'].map(mapping)\n",
        "\n",
        "hs_and_offensive_lang_df\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "zqporNwsYFS3"
      },
      "source": [
        "### Clean the dataset again."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "5fdIWLLVYFS3",
        "outputId": "393ad7e2-c789-4d0d-9249-ab4df6191293"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 29,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "hs_and_offensive_lang_df.duplicated().sum()"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aQaimF_RYFS3"
      },
      "source": [
        "## 1.4 | Combine the different data frames."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "sdUm3M3kYFS3",
        "outputId": "16a850aa-1c78-43a4-9361-4265447b03d0"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92967</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92968</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92969</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92970</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92971</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>92972 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "92967  you's a muthaf***in lie &#8220;@LifeAsKing: @2...      1\n",
              "92968  you've gone and broke the wrong heart baby, an...      0\n",
              "92969  young buck wanna eat!!.. dat nigguh like I ain...      1\n",
              "92970              youu got wild bitches tellin you lies      1\n",
              "92971  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...      0\n",
              "\n",
              "[92972 rows x 2 columns]"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Make a single data frame for all the datasets.\n",
        "df = pd.concat([hate_speech_fil_df, toxic_tweets_df, hs_and_offensive_lang_df], ignore_index=True)\n",
        "\n",
        "df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "dU05fRcUYFS3"
      },
      "source": [
        "### Clean the data."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ReepG4_zYFS3"
      },
      "source": [
        "After countless experiments, removing the substrings that starts with @, http, and, https reduces the accuracy rate on sufficiently large data sets. For now, we will comment this subsequent code out.\n",
        "\n",
        "Adding more to this, we initially thought that urls and twitter handler names would be useless in classifiying the text since they just 'bloat' the extracted tokens by the CountVectorizer. But we were wrong, the total model accuracy would decrease by 2% to 3% if we remove these said substrings. Our hypothesis is that these subtrings adds more 'context' in classifying the text. For instance, people with '@deeznuts' or any other 'toxic' twitter handler names are more likely to tweet toxic speeches."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "SXTpv5KrYFS3"
      },
      "outputs": [],
      "source": [
        "# # Remove the substrings that starts with @, http, and https.\n",
        "# # They are irrelevant to the classification.\n",
        "\n",
        "# import re\n",
        "\n",
        "# for index, row in df.iterrows():\n",
        "#     df.loc[index, 'text'] = re.sub(r'@\\S+', '', df.loc[index, 'text'])\n",
        "#     df.loc[index, 'text'] = re.sub(r'http\\S+', '', df.loc[index, 'text'])\n",
        "#     df.loc[index, 'text'] = re.sub(r'https\\S+', '', df.loc[index, 'text'])\n",
        "\n",
        "# df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "rBBFaOujYFS4",
        "outputId": "da71692a-9c7f-42b8-a343-11cf6ac1faba"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "23492"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for duplicates.\n",
        "df.duplicated().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "HXNMO14aYFS4",
        "outputId": "2d441524-f22d-48ab-eeb1-31603f2aedaf"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>68189</th>\n",
              "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68190</th>\n",
              "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68191</th>\n",
              "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68192</th>\n",
              "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>68193</th>\n",
              "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92966</th>\n",
              "      <td>you're such a retard i hope you get type 2 dia...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92967</th>\n",
              "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92969</th>\n",
              "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92970</th>\n",
              "      <td>youu got wild bitches tellin you lies</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>92971</th>\n",
              "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>23492 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "68189  !!! RT @mayasolovely: As a woman you shouldn't...      0\n",
              "68190  !!!!! RT @mleew17: boy dats cold...tyga dwn ba...      1\n",
              "68191  !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...      1\n",
              "68192  !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...      1\n",
              "68193  !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...      1\n",
              "...                                                  ...    ...\n",
              "92966  you're such a retard i hope you get type 2 dia...      1\n",
              "92967  you's a muthaf***in lie &#8220;@LifeAsKing: @2...      1\n",
              "92969  young buck wanna eat!!.. dat nigguh like I ain...      1\n",
              "92970              youu got wild bitches tellin you lies      1\n",
              "92971  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...      0\n",
              "\n",
              "[23492 rows x 2 columns]"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the duplicates.\n",
        "df[df.duplicated()]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "tV_554QHYFS4",
        "outputId": "86afd3e3-9179-4e28-cfb8-bd1a0ea9ca8a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69475</th>\n",
              "      <td>when baylor wins im talking so much trash</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69476</th>\n",
              "      <td>who's downie like a brownie 4 brunch? Need mim...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69477</th>\n",
              "      <td>yea so about @N_tel 's new friend.. all my fri...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69478</th>\n",
              "      <td>you know what they say, the early bird gets th...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69479</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69480 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "69475          when baylor wins im talking so much trash      0\n",
              "69476  who's downie like a brownie 4 brunch? Need mim...      0\n",
              "69477  yea so about @N_tel 's new friend.. all my fri...      0\n",
              "69478  you know what they say, the early bird gets th...      0\n",
              "69479  you've gone and broke the wrong heart baby, an...      0\n",
              "\n",
              "[69480 rows x 2 columns]"
            ]
          },
          "execution_count": 34,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Drop the duplicates.\n",
        "df.drop_duplicates(inplace=True)\n",
        "\n",
        "# Reset the index.\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "biABX9z6YFS4",
        "outputId": "86993c0d-6d21-4c60-90ee-0bb8d1666f8c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "execution_count": 35,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Verify that there are no duplicates.\n",
        "df.duplicated().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "bigDTdATYFS4",
        "outputId": "221c1914-3c1d-4e6e-95b0-30139586bd18"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [text, label]\n",
              "Index: []"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for texts that only contain number strings.\n",
        "df[df['text'].str.isnumeric()]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "M_8DCpFAfEWP"
      },
      "source": [
        "The code snippets below are meant to remove the 'empty' and irrelevant rows after removing those specified substrings above (e.g., handler names, urls, etc.) since there are rows in the original dataset that only contains handler names, urls, and the like. We decided to not delete these code snippets just yet even if they don't explicitly do anything useful (for future reference or change of mind in the data cleaning procedures)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "0qZLnoIRYFS4",
        "outputId": "b84878a8-add5-4625-c62a-0c83510656e2"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [text, label]\n",
              "Index: []"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Check for empty strings.\n",
        "df[df['text'] == '']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "HGZANkrsYFS4",
        "outputId": "ac5e2978-1a38-4efc-f5b4-c83a83750ba4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69475</th>\n",
              "      <td>when baylor wins im talking so much trash</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69476</th>\n",
              "      <td>who's downie like a brownie 4 brunch? Need mim...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69477</th>\n",
              "      <td>yea so about @N_tel 's new friend.. all my fri...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69478</th>\n",
              "      <td>you know what they say, the early bird gets th...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69479</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69480 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "69475          when baylor wins im talking so much trash      0\n",
              "69476  who's downie like a brownie 4 brunch? Need mim...      0\n",
              "69477  yea so about @N_tel 's new friend.. all my fri...      0\n",
              "69478  you know what they say, the early bird gets th...      0\n",
              "69479  you've gone and broke the wrong heart baby, an...      0\n",
              "\n",
              "[69480 rows x 2 columns]"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Remove the rows with empty strings.\n",
        "df.drop(df[df['text'] == ''].index, inplace=True)\n",
        "\n",
        "# Reset the index.\n",
        "df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "PNiu9NrrYFS5",
        "outputId": "e1880467-a73f-4198-c444-0f0a44a879f5"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "Empty DataFrame\n",
              "Columns: [text, label]\n",
              "Index: []"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Verify that there are no empty strings.\n",
        "df[df['text'] == '']"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "eos-FslUYqdR"
      },
      "source": [
        "# [ AI MODEL ]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "V51a4EWFYFS5"
      },
      "source": [
        "## Creating a Train and Test sets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "kWcd2H8RYFS5",
        "outputId": "4e684704-3fc2-4adb-f17a-490dec01c3f1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(69480,)\n",
            "(69480,)\n"
          ]
        }
      ],
      "source": [
        "X = df['text']\n",
        "y = df['label']\n",
        "\n",
        "print(X.shape)\n",
        "print(y.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "HuXycsf8YFS5",
        "outputId": "fd52e21a-2a22-4aaf-dbe4-b30b00753ca1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(52110,)\n",
            "(17370,)\n",
            "(52110,)\n",
            "(17370,)\n"
          ]
        }
      ],
      "source": [
        "# Split X and y into training and testing sets.\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=69)\n",
        "\n",
        "print(X_train.shape)\n",
        "print(X_test.shape)\n",
        "print(y_train.shape)\n",
        "print(y_test.shape)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "RY4aNlZdYFS5"
      },
      "source": [
        "## Vectorizing the dataset (w/o optimized params)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "VRrUA34WYFS5"
      },
      "outputs": [],
      "source": [
        "# Instantiate the CountVectorizer.\n",
        "\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "cvect = CountVectorizer()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {
        "id": "XDoyudFUYFS5",
        "outputId": "ff754b25-939f-4361-f8a8-4715e63d0a14"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>CountVectorizer()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">CountVectorizer</label><div class=\"sk-toggleable__content\"><pre>CountVectorizer()</pre></div></div></div></div></div>"
            ],
            "text/plain": [
              "CountVectorizer()"
            ]
          },
          "execution_count": 43,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Learn the training data vocabulary, then use it to create a document-term matrix.\n",
        "\n",
        "cvect.fit(X_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "wRR5lFZOYFS5",
        "outputId": "ca0fdd9a-72e7-42de-f478-a1290c88709d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['00' '000' '0000000000000' ... 'ð¾ñ' 'ó¾' 'ø¹ù']\n",
            "len: 69692\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0000000000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>000001</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69687</th>\n",
              "      <td>ð¾ð½ð</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69688</th>\n",
              "      <td>ð¾ð½ðµð</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69689</th>\n",
              "      <td>ð¾ñ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69690</th>\n",
              "      <td>ó¾</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69691</th>\n",
              "      <td>ø¹ù</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69692 rows × 1 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                   0\n",
              "0                 00\n",
              "1                000\n",
              "2      0000000000000\n",
              "3             000001\n",
              "4                002\n",
              "...              ...\n",
              "69687          ð¾ð½ð\n",
              "69688        ð¾ð½ðµð\n",
              "69689            ð¾ñ\n",
              "69690             ó¾\n",
              "69691            ø¹ù\n",
              "\n",
              "[69692 rows x 1 columns]"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Examine the fitted vocabulary.\n",
        "\n",
        "feature_names = cvect.get_feature_names_out()\n",
        "print(feature_names)\n",
        "print('len:', len(feature_names))\n",
        "pd.DataFrame(feature_names)\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "pm2I2IXygHLK"
      },
      "source": [
        "There are a lot of extracted unique tokens. This would mean a larger runtime in fitting the model later on."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "LheO6EnUYFS6"
      },
      "outputs": [],
      "source": [
        "# Transform the training data into a document-term matrix.\n",
        "\n",
        "# X_train_dtm = vect.transform(X_train)\n",
        "X_train_dtm = cvect.fit_transform(X_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "id": "x0ejkQuTYFS6",
        "outputId": "8c1fc91c-1f19-4d0e-c112-b3ef65f0e9da"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<52110x69692 sparse matrix of type '<class 'numpy.int64'>'\n",
              "\twith 653595 stored elements in Compressed Sparse Row format>"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Examine the document-term matrix.\n",
        "\n",
        "X_train_dtm"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "kQ8voyMKgtQ6"
      },
      "source": [
        "Converting it to a dtm is really useful. We have found out that if we did not do this, we will need approximately 45+GB memory in building the AI model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "id": "tkfRAbM3YFS6",
        "outputId": "ffec23f2-c829-4220-d61d-eea619b56215"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<17370x69692 sparse matrix of type '<class 'numpy.int64'>'\n",
              "\twith 202486 stored elements in Compressed Sparse Row format>"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Transform the testing data (using fitted vocabulary) into a document-term matrix.\n",
        "\n",
        "X_test_dtm = cvect.transform(X_test)\n",
        "X_test_dtm"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jP4u_kAsYFS6"
      },
      "source": [
        "# 2 | Building and Evaluating the MNB model"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5r8Nxl_PkVMX"
      },
      "source": [
        "We chose Monomial Naive Bayes classifier because it can be effective in classifying toxic speeches for several reasons:\n",
        "\n",
        "1. **Simplicity and efficiency:** Monomial Naive Bayes classifiers are relatively simple and computationally efficient. They are based on the assumption of feature independence, which means they treat each feature (word) in isolation, disregarding any relationships or dependencies between them. This assumption allows for faster training and classification compared to more complex models.\n",
        "\n",
        "2. **Bag-of-words representation:** Monomial Naive Bayes classifiers commonly use a bag-of-words representation, where the order of words is disregarded and only the occurrence or frequency of each word is considered. This approach is suitable for toxic speech classification because toxic speeches often contain specific keywords or phrases that can be indicative of their toxicity. By focusing on individual words, the classifier can capture these patterns effectively.\n",
        "\n",
        "3. **Text categorization:** Naive Bayes classifiers are particularly well-suited for text categorization tasks like toxic speech classification. They calculate the probability of a given speech belonging to a particular class (e.g., toxic or non-toxic) based on the occurrence of words in the speech. This makes them effective in identifying toxic patterns by analyzing the presence or absence of specific words associated with toxic speech.\n",
        "\n",
        "4. **Reasonable performance:** Despite their simplicity, Naive Bayes classifiers have shown good performance in various text classification tasks, including toxic speech detection. They can achieve reasonably high accuracy when trained on appropriately labeled data. Moreover, they can handle large feature spaces (vocabulary size) efficiently, making them suitable for tasks involving a wide range of words.\n",
        "\n",
        "It's worth noting that while Monomial Naive Bayes classifiers can be effective in toxic speech classification, they have limitations. For example, they assume independence between features, which may not always hold in the context of language. Although, we can use a trick to let this model 'understand' the context of specific substrings by increasing the n-gram param value. Additionally, they may struggle with capturing more complex linguistic structures or context beyond individual words. Nonetheless, with proper feature engineering and training data, they can be a valuable tool in identifying toxic speeches.\n",
        "\n",
        "For testing the model, we use 20% of the original dataset while 80% was used to train the model."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "AvUzNZxfYFS6"
      },
      "source": [
        "## Building the MNB model (w/o optimized params)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "aTYJvduQYFS6"
      },
      "outputs": [],
      "source": [
        "# Import and instantiate a Multinomial Naive Bayes model.\n",
        "\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "mnb = MultinomialNB()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "RZAzniz0YFS6",
        "outputId": "88477d0e-07ba-4bd9-b9df-96a81ee40dcd"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div>"
            ],
            "text/plain": [
              "MultinomialNB()"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Train the model using X_train_dtm and the fit() method.\n",
        "\n",
        "mnb.fit(X_train_dtm, y_train)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "HeM4mTlcYFS7"
      },
      "source": [
        "## Evaluating the MNB model (baseline performance)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "5OvYLt1ZYFS7"
      },
      "outputs": [],
      "source": [
        "# Make class predictions for the X_test_dtm. Store it in the y_pred_class variable.\n",
        "\n",
        "y_pred_class = mnb.predict(X_test_dtm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "id": "HMy8NkXuYFS7",
        "outputId": "88b7e328-3f6f-4365-9502-13c608999c6d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.8637881404720783"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Calculate the accuracy of the predictions.\n",
        "\n",
        "from sklearn import metrics\n",
        "\n",
        "metrics.accuracy_score(y_test, y_pred_class)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "id": "5kpRliwMYFS7",
        "outputId": "c920ebb7-544c-4b33-911e-f578312f75b1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[8391, 1409],\n",
              "       [ 957, 6613]], dtype=int64)"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAGwCAYAAADITjAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyDElEQVR4nO3deZyNdf/H8feZ7cxYZtCYLE2WpkKMIbcsWZIiKqGQ7JEWqbgRZUvWynqHomwhYrjbaEG2SmimEWOJsZXB3MNMaMacmev3h5+T05lhzjjjco3X8/E4j+Z8r+/5Xp9Lh/dc1/dabIZhGAIAwGJ8zC4AAIC8IMAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAl+ZldQH7ISDpgdglAvgoq08DsEoB85Tj/+xX7sAcGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACzJ9AB7/PHHNW7cOLf2t956S0888YQJFQEArMD0AFu/fr1atmzp1t68eXNt2LDBhIoAAFZgeoCdOXNGAQEBbu3+/v5KTU01oSIAgBWYHmBVq1bVkiVL3No//vhjValSxYSKAABW4Gd2AUOHDlXbtm21f/9+NWnSRJK0Zs0aLV68WJ988onJ1QEArlemB9ijjz6qlStXasyYMVq2bJmCgoIUGRmpb7/9Vo0aNTK7PADAdcpmGIZhdhHelpF0wOwSgHwVVKaB2SUA+cpx/vcr9jF9DgwAgLww5RBiiRIltHfvXoWGhqp48eKy2Ww59k1OTr6GlQEArMKUAJs0aZKKFi3q/PlyAQYAQHaYAwMsiDkwFHSWmAP74IMPsm13OBwaPHjwNa4GAGAVpgdY//791bZtW5e5rt27d6t27dpaunSpiZUBAK5npgdYTEyMjh8/rmrVqumbb77Ru+++q5o1a6pq1aqKjY01uzwAwHXK9AuZK1SooA0bNuiVV15R8+bN5evrq/nz56tDhw5mlwYAuI6ZvgcmSZ9//rkWL16sevXqqVixYpo1a5b++OMPs8sCAFzHTA+w3r17q127dho4cKA2bNiguLg42e12VatWjTkwAECOTD+NvmrVqlq4cKGqV6/u0v7uu+9q0KBBOnPmjMdjcho9CjpOo0dBl5vT6E0PsPT0dNnt9myX7dmzR3feeafHYxJgKOgIMBR0uQkw00/isNvtOn36tD744APFx8fLZrOpcuXKevrpp/MUXgCAG4Ppe2Dbtm1Ts2bNFBQUpNq1a8swDG3btk1//fWXvv76a9WsWdPjMdkDQ0HHHhgKOkscQmzQoIEiIiI0a9Ys+fld2CF0OBzq2bOnDhw4oA0bNng8JgGGgo4AQ0FniQALCgpSTEyMKlWq5NK+a9cu1apVS+fOnfN4TAIMBR0BhoLOEvdCDA4O1uHDh93ajxw54rxjPfKXw5Gpqe/PU7PHu+nu+1qp+RPdNePDhcrKynL2efeDj/TIk730r/sfU73mT6jnS4MVt3O3yziHj/6hvoPfUIOW7XXPA23Uf+gYJSWfcunz3rzFeqp3P9Vq8pjqNnv8mmwfkJ0iRQrrnbdHav++Lfoz5TdtXP9f1br777OhP5g9SY7zv7u8Nm/8zLm8XLlb3JZffLVt+7AZm3TDMS3A5s+fr/T0dLVv315PP/20lixZoiNHjujo0aP6+OOP1bNnTz355JNmlXdD+WDhUi1d+aWG9Hteny56X/2e76E5i5Zr4bJPnX3Kh5fVkH7PK3r+DM2f/rbKlLpZz7zympJPnZYknfsrTc+88ppssumDqeO0YOY7yshwqM/AES5BmJHhULP7Gqh965bXejMBF++/97aaNm2gbt37KqpmU33z7Xp9tfpjlSlTytln9eq1Khse5Xw9/Ghn57IjR/5wWVY2PEojRr6lM2fOavXqtWZs0g3HtLMQu3fvrubNm+vtt9+WzWZTly5d5HA4JEn+/v567rnnNG7cOLPKu6H88utu3degjhrVqy1JKlv6Zn35zXrt3L3P2aflg/e5fGZg316K/vwr7d2foDq1aigmbqf+SDyhZXP/oyKFC0uSRg15RfUfaqct239R3X/VkCT16XnhH4CVX3xzLTYNyFZgYKDatG6hNm17aOOmLZKkN0ZN1KOPNtezvbto2PAJkqT08+d1/PjJbMfIyspyW9aq1UNa+smnOnvW86kPeM60PbCLU28BAQGaMmWKTp06pdjYWMXExCg5OVmTJk3K8foweFfNyLu0ZVusDh4+Kknave+Afo7bqYZ1/5Vt/4yMDH3y31UqWqSw7oyo6Gyz2aQAf39nP7s9QD4+Pvo5bmf+bwTgAT8/X/n5+SktLd2lPe2vNNWv9/f3vlHDuvrj6C/atXOjZs6YoJIlb8pxzJo1qqlGVFXNmfNxvtUNV6ZeB3bpk5gLFSqkatWqmVjNjevpTk/ozzNn9UjHZ+Tr46PMrCz1faarWjzQ2KXfd5u3aMDwcUpLS1fJm0ro/cmjVbxYiCQp8q5KCgoM1MTpH+qlZ7vJMKRJ0z9UVlaWkv6XnM1aAfOcOXNWP/ywTa8NeUnxu/fp+PGT6tDhMdWuXUP7fkuQJK3+ap2WL/9chw4fVYXyt2rEiAH65uulqn3PQzp//rzbmN27P6ld8Xv1w4/brvXm3LBMDbBu3bpdcS8rOjr6ssvT09OVnu76W5TPZe7uAXer1qzX51+v1fgRAxVRoZx27zug8VPeU1hoCbVq8YCzX+2a1bV87rs6dTpFyz5brX8PHatFsybrpuLFVKJ4Mb0zaohGvf0fLVz2qXx8bHqoaWNVuTNCPj6mnysEuOnava9mv/+Ojhz6WQ6HQzExO7T44xWqUePCL9KffPL3HPDOnXu0bfsvOvDbFrVocb9WrlzlMlZgYKCe7PCYRo+Zck234UZnaoAVLVpUQUFBVzXG2LFjNXLkSJe21wf01bCBL13VuDeSd979QD07tVOLpo0lSXfcVkHHEk9o9oKlLgFWKChQt95SRrfeUkbVq1ZWi/ZPK/qzr9SrS3tJUv177tbqT+bo1OkU+fr6KrhoETV6pKOaly6V3WoBUx04cEhNmj6uQoWCFBxcVImJJ7Ro4QwdTDiSbf/ExBM6dOh33R5RwW1Z27YtVahQkBZ89El+l41LmBpgU6dOVVhY2FWNMXjwYPXr18+lzefPK18/gL+lpaXL5mNzafPx8VHWFS4RNAxD5zMy3NovHlbcsj1WyadO675763ivWMDLzp37S+fO/aVixUL04AON9Org0dn2K1GiuMLDS+tY4gm3ZT26ddBnn3+jpCQOl19LpgXYpfNfV8Nut7sdLsw4n+SVsW8Ujevfo1nzPlbpm8MUUaGc4vf+pvlLotW65YOSLpwi//68j3XfvfeoZGgJnU75Ux9Hf67jJ5PU7L6/L6hd8cXXqlguXMWLheiXnbs1bvJMdWnfWhXK3eLscyzxhFJS/9Sx4yeUmZml3Xv3S5JuvaWMChW6ur1xwBMPPtBINptNe/buV8Rt5TVu3FDt3btfc+ctUeHChTR8aH9Fr/hSxxKPq3y5cL056lUlJZ1yO3x4223l1aBBHT1yySn2uDZMCzCTbwCCSwx55TlNmzVfb779rpJPnVbJ0BJ6olULPde9oyTJ18dHCYeO6NNV3+pUSoqKBQerauU7NG/6W4qoWM45zsHDRzV55lylpP6psqVv1jNdO6hL+9Yu6/rP7AX676pvne8f795HkvThtPGqXTPyGmwtcEFwSLBGj3pVt9xSWsnJpxW94ksNHTZeDodDfn5+qlq1kjp1elzFigXr2LET+m7993ryqed05sxZl3G6d+ug339P1NffrDdpS25cpt1Kav369apfv77z/ofexK2kUNBxKykUdNf141QaNWpk1qoBAAUA5zcDACyJAAMAWBIBBgCwJNMDzNfXVydOuF9X8b///U++vr4mVAQAsALTAyynkyDT09MVEBBwjasBAFiFaWchTp06VdKFC5pnz56tIkWKOJdlZmZqw4YNbk9pBgDgItMCbNKkSZIu7IHNnDnT5XBhQECAypcvr5kzZ5pVHgDgOmdagCUkXHhkwX333afo6GgVL17crFIAABZk6s18JWndunXOny/Oh3nrPokAgILL9JM4JGn+/PmqVq2agoKCFBQUpMjISC1YsMDssgAA1zHT98AmTpyooUOHqk+fPqpfv74Mw9DmzZv17LPPKikpSa+88orZJQIArkOm3cz3ogoVKmjkyJHq0qWLS/u8efM0YsQI51yZJ7iZLwo6buaLgi43N/M1/RDisWPHVK9ePbf2evXq6dixYyZUBACwAtMDLCIiQkuXLnVrX7JkiW6//XYTKgIAWIHpc2AjR45U+/bttWHDBtWvX182m02bNm3SmjVrsg02AACk62APrG3bttqyZYtCQ0O1cuVKRUdHKzQ0VD/99JNat2595QEAADck00/iyA+cxIGCjpM4UNBZ4iQOAADywrQ5MB8fnyveccNms8nhcFyjigAAVmJagK1YsSLHZd9//72mTZuW46NWAAAwLcBatWrl1rZ7924NHjxYn332mZ566imNGjXKhMoAAFZwXcyB/fHHH+rVq5ciIyPlcDgUGxurefPm6dZbbzW7NADAdcrUAEtJSdGgQYMUERGhnTt3as2aNfrss89UtWpVM8sCAFiAaYcQJ0yYoPHjx6tUqVJavHhxtocUAQDIiWnXgfn4+CgoKEhNmzZ1eRrzP0VHR3s8NteBoaDjOjAUdLm5Dsy0PbAuXbrw4EoAQJ5xJw7AgtgDQ0HHnTgAAAUWAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACwpV3fiiIuLy/WAkZGReS4GAIDcylWARUVFyWaz5fiAyYvLbDabMjMzvVogAADZyVWAJSQk5HcdAAB4JFcBVq5cufyuAwAAj+TpJI4FCxaofv36KlOmjA4dOiRJmjx5sv773/96tTgAAHLicYDNmDFD/fr1U4sWLXT69GnnnFexYsU0efJkb9cHAEC2PA6wadOmadasWXrttddcHkRZq1Yt7dixw6vFAQCQE48DLCEhQTVq1HBrt9vtOnv2rFeKAgDgSjwOsAoVKig2NtatfdWqVapSpYo3agIA4IpydRbipQYMGKAXXnhBaWlpMgxDP/30kxYvXqyxY8dq9uzZ+VEjAABuPA6w7t27y+FwaODAgTp37pw6duyosmXLasqUKerQoUN+1AgAgBubkdPtNXIhKSlJWVlZCgsL82ZNVy0j6YDZJQD5KqhMA7NLAPKV4/zvV+zj8R7YRSdOnNCePXtks9lks9lUsmTJvA4FAIDHPD6JIzU1VZ07d1aZMmXUqFEjNWzYUGXKlFGnTp2UkpKSHzUCAODG4wDr2bOntmzZoi+++EKnT59WSkqKPv/8c23btk29evXKjxoBAHDj8RxY4cKF9dVXX+nee+91ad+4caOaN29+XVwLxhwYCjrmwFDQ5WYOzOM9sJtuukkhISFu7SEhISpevLinwwEAkCceB9jrr7+ufv366dixY862xMREDRgwQEOHDvVqcQAA5CRXZyHWqFFDNpvN+X7fvn0qV66cbr31VknS4cOHZbfbdfLkSfXu3Tt/KgUA4BK5CrDHHnssn8sAAMAzV3Uh8/WKkzhQ0HESBwq6fDmJAwCA64HHd+LIzMzUpEmTtHTpUh0+fFjnz593WZ6cnOy14gAAyInHe2AjR47UxIkT1a5dO6WkpKhfv35q06aNfHx8NGLEiHwoEQAAdx7Pgd12222aOnWqWrZsqaJFiyo2NtbZ9uOPP2rRokX5VWuuMQeGgo45MBR0+TIHlpiYqGrVqkmSihQp4rz/4cMPP6wvvvjC0+EAAMgTjwPslltucV7EHBERoa+//lqStHXrVtntdu9WBwBADjwOsNatW2vNmjWSpJdeeklDhw7V7bffri5duqhHjx5eLxAAgOxc9XVgW7Zs0ebNmxUREaFHH33UW3VdFebAUNAxB4aCLjdzYF67kPn48eN67733NGzYMG8Md1UIMBR0BBgKumt6IXNiYqJGjhzpreEAALgs7sQBALAkAgwAYEkEGADAknJ9L8R+/fpddvnJkyevuhgAAHIr1wEWExNzxT4NGza8qmIAAMitXAfYunXr8rMOAAA8whwYAMCSCDAAgCURYAAASyLAAACWRIABACwpV2chxsXF5XrAyMjIPBcDAEBu5SrAoqKiZLPZZBiGbDbbZftmZmZ6pTAAAC4nV4cQExISdODAASUkJGj58uWqUKGCpk+frpiYGMXExGj69Om67bbbtHz58vyuFwAASXl4Hljt2rU1YsQItWjRwqX9yy+/1NChQ7V9+3avFpgXPA8MBR3PA0NBly/PA9uxY4cqVKjg1l6hQgXt2rXL0+EAAMgTjwOscuXKevPNN5WWluZsS09P15tvvqnKlSt7tTgAAHKS63shXjRz5kw98sgjCg8PV/Xq1SVJv/zyi2w2mz7//HOvFwgAQHY8ngOTpHPnzumjjz7S7t27ZRiGqlSpoo4dO6pw4cL5UaPHmANDQcccGAq63MyB5SnArncEGAo6AgwFXW4CzONDiJK0YMECvffeezpw4IB++OEHlStXTpMmTVLFihXVqlWrvAzpVeERLc0uAchXqWMeMrsEwHQen8QxY8YM9evXTw899JBOnTrlvHC5ePHimjx5srfrAwAgWx4H2LRp0zRr1iy99tpr8vP7eweuVq1a2rFjh1eLAwAgJx4HWEJCgmrUqOHWbrfbdfbsWa8UBQDAlXgcYBUqVFBsbKxb+6pVq1SlShVv1AQAwBV5fBLHgAED9MILLygtLU2GYeinn37S4sWLNXbsWM2ePTs/agQAwI3HAda9e3c5HA4NHDhQ586dU8eOHVW2bFlNmTJFHTp0yI8aAQBwc1XXgSUlJSkrK0thYWHerOmqlSrGLa1QsB14vb7ZJQD5qtC/r3xEz+M5sCZNmuj06dOSpNDQUGd4paamqkmTJp4OBwBAnngcYN99953Onz/v1p6WlqaNGzd6pSgAAK4k13NgcXFxzp937dqlxMRE5/vMzEytXr1aZcuW9W51AADkINcBFhUVJZvNJpvNlu2hwqCgIE2bNs2rxQEAkJNcB1hCQoIMw1DFihX1008/qWTJks5lAQEBCgsLk6+vb74UCQDAP+U6wMqVKydJysrKyrdiAADILY9P4hg7dqw+/PBDt/YPP/xQ48eP90pRAABciccB9t5776lSpUpu7XfddZdmzpzplaIAALgSjwMsMTFRpUuXdmsvWbKkjh075pWiAAC4Eo8DLDw8XJs3b3Zr37x5s8qUKeOVogAAuBKP74XYs2dPvfzyy8rIyHCeTr9mzRoNHDhQ/fv393qBAABkx+MAGzhwoJKTk/X8888778gRGBioQYMGafDgwV4vEACA7OT5Zr5nzpxRfHy8goKCdPvtt8tut3u7tjzjZr4o6LiZLwq63NzM1+M9sIuKFCmif/3rX3n9OAAAVyVXAdamTRvNnTtXwcHBatOmzWX7RkdHe6UwAAAuJ1cBFhISIpvN5vwZAACzXdUDLa9XzIGhoGMODAVdvjzQEgCA60GuDiHWqFHDeQjxSn7++eerKggAgNzIVYA99thjzp/T0tI0ffp0ValSRXXr1pUk/fjjj9q5c6eef/75fCkSAIB/ylWADR8+3Plzz5491bdvX40aNcqtz5EjR7xbHQAAOfB4DuyTTz5Rly5d3No7deqk5cuXe6UoAACuxOMACwoK0qZNm9zaN23apMDAQK8UBQDAlXh8J46XX35Zzz33nLZv3646depIujAH9uGHH2rYsGFeLxAAgOx4HGCvvvqqKlasqClTpmjRokWSpMqVK2vu3Llq166d1wsEACA7eboXYrt27QgrAICp8nQh8+nTpzV79mwNGTJEycnJki5c//X77797tTgAAHLi8R5YXFycmjZtqpCQEB08eFA9e/ZUiRIltGLFCh06dEjz58/PjzoBAHDh8R5Yv3791K1bN+3bt8/lrMOHHnpIGzZs8GpxAADkxOMA27p1q3r37u3WXrZsWSUmJnqlKAAArsTjAAsMDFRqaqpb+549e1SyZEmvFAUAwJV4HGCtWrXSG2+8oYyMDEmSzWbT4cOH9eqrr6pt27ZeLxAAgOx4HGBvv/22Tp48qbCwMP31119q1KiRIiIiVLRoUY0ePTo/agQAwI3HZyEGBwdr06ZNWrt2rX7++WdlZWWpZs2aatq0aX7UBwBAtjwKMIfDocDAQMXGxqpJkyZq0qRJftUFAMBleXQI0c/PT+XKlVNmZmZ+1QMAQK54PAf2+uuva/Dgwc47cAAAYAaP58CmTp2q3377TWXKlFG5cuVUuHBhl+U///yz14oDACAnHgdYq1atZLPZ8qMWAAByzeMAGzFiRD6UAQCAZ3I9B3bu3Dm98MILKlu2rMLCwtSxY0clJSXlZ20AAOQo1wE2fPhwzZ07Vy1btlSHDh30zTff6LnnnsvP2gAAyFGuDyFGR0frgw8+UIcOHSRJnTp1Uv369ZWZmSlfX998KxAAgOzkeg/syJEjatCggfN97dq15efnpz/++CNfCgMA4HJyHWCZmZkKCAhwafPz85PD4fB6UQAAXEmuDyEahqFu3brJbrc729LS0vTss8+6XAsWHR3t3QoBAMhGrgOsa9eubm2dOnXyajEAAORWrgNszpw5+VkHAAAe8fheiAAAXA8IMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACzJ9ABLSEjQvn373Nr37dungwcPXvuCAACWYHqAdevWTd9//71b+5YtW9StW7drXxAAwBJMD7CYmBjVr1/frb1OnTqKjY299gUBACzB9ACz2Wz6888/3dpTUlKUmZlpQkUAACswPcAaNGigsWPHuoRVZmamxo4dq3vvvdfEygAA1zM/swuYMGGCGjZsqDvvvFMNGjSQJG3cuFGpqalau3atydUBAK5Xpu+BValSRXFxcWrXrp1OnDihP//8U126dNHu3btVtWpVs8sDAFynTN8Dk6QyZcpozJgxZpcBALAQUwIsLi5OVatWlY+Pj+Li4i7bNzIy8hpVBQCwElMCLCoqSomJiQoLC1NUVJRsNpsMw3DrZ7PZOBMRAJAtUwIsISFBJUuWdP4Mc9WpV0vP9+2hyOp3qVTpMHV7qo9Wf7Em274TJo1Ql+7tNXTwWM2aMd/ZHhDgr+FvDtRjbVsqKNCujRt+1Kv939CxP447+1SrXkWvj+ivqJpVlZmZpS8+/VrDXxuvc2fP5fs2ArYixeTf8HH5Vqgq+fnLOHVc6V/Nk3H80N99SpSWf8O28g2/Q7L5KCvpd53/7D0ZfyZLknwjG8qv8j3yCbtVNnuQzk17UUr/y2U9AY/1kU9YuGyFgqW0s8o8FK+MDctknE25ptt7IzAlwMqVK5ftz/+U3V4ZvK9QoSDt3LFHHy9coQ8XTM2xX/OW96tmrUiXULpo1NgheqB5Yz3bo79OnTqtEW8O1IIlM/Rgo8eVlZWlm0uV1NKVH+jT6NUaMmCUihYtojfGDtbU6WPUs+vL+bh1gCR7IdmffFVZR/YoffkUGedSZStWUkr7+5cnW0hJBT45SI4dm5T2/X+l9L9ku6m0jMyMv/v4BSgz4VdlJvyqgIZts11V1pE9cmz5UsbZ07IVKS7/Rk8o4NHnlL54XL5v5o3G9JM4OnfurBkzZqhIkSIu7QcPHlTnzp21ceNGkyq7caz9dqPWfnv5P+dSpcM0ZsLrerJtL320dKbLsqLBRfRk5zZ6sfer2rj+B0nSC88M1M8716lh47r6bu1mPdCssRwZDr367zecv5gMHjBKazauUPkKt+pgwuH82ThAkn/th2T8mazzq+c424zU/7n2adBamQd2KGPDsr/7pCS59HH8/K0kySf8zhzX5dj+zSXrSJbjp1UKeOwFycdXymJKxJtMP41+165dqlatmjZv3uxsmzdvnqpXr66bb77ZxMpwkc1m03/eG6/p0z7Unt2/uS2PjLpLAQEB+m7t3/8Pjyee1O74ffrXPTUkSXZ7gM6fz3DZq077K02SdE/dmvm8BbjR+UZUV1biIQU88qyCnp+owM7D5FutwSU9bPKtGKmsU8dlb/uygp6fKPtTQ+QbEXV1Kw4sLN/KdZT1+37CKx+YHmBbtmxR+/bt1aRJEw0ZMkRPPPGE+vTpo0mTJmnZsmVX/Hx6erpSU1NdXoaRdQ0qv3H0ebmnHI5MzZ65INvlYWGhSk8/r5SUVJf2kyf+p5JhoZKkTRu2KOzmUD3/Yg/5+/srJCRYg4e9cuHzN5fM3w3ADc8WUlJ+UY1lnDqutGWT5PjlOwU0eVK+Vepe6FCoqGwBgfK/5yFlHtyptE8mKXNfjAJaPS+fW+7weH3+Ddsq6KV3VajPFNmCSyh95X+8vEWQroNDiH5+fho3bpzsdrtGjRolPz8/rV+/XnXr1s3V58eOHauRI0e6tBW236Qigfyj6A2R1auo17Od9UCj7I/3X86lZ5fu2f2b+j43WCNHD9KQ4a8oMzNLH7y3QCeOn1RWJr9wIJ/ZbMpKPKiMTSskSY4TR2QLLSu/qMbK3PWDbDabJCnzt1jnIUDHySPyKXOb/Ko30vmjez1aXcbWr+TYsUm24JvkX/cR2Vs8rfTonOeXkTem74FlZGSof//+Gj9+vAYPHqy6deuqdevW+vLLL3P1+cGDByslJcXlVdh+Uz5XfeO4p14thZa8Sdt/XaujSTt0NGmHwm8tqxFvDtTWuAvzASdOJMluD1BISLDLZ0NLllDSyb/nGVYs+0KRdzZUVOXGqlyxrt4e965uCi2hw4eOXtNtwo3HOJsi43/HXNv+d0y2oiUu/PzXGRmZDmX97w/XPsnHZAvOw78nf52Rceq4sg7t0vnP35dvxUj5lK6Y5/qRPdP3wGrVqqVz587pu+++U506dWQYhiZMmKA2bdqoR48emj59+mU/b7fbZbfbXdpsNtNzucBY9vGn2vjdDy5ti5fP0rIln+rjhdGSpLjYnTp//rwa3VdPn65cLenCYcFKlW/XqGFvu415MdSe7NRG6WnpWv+d+/PgAG/K+v032Uq4zqnbit/894kcWZnKSjwon+Klcu6TV7b//6+f/9WNAzfXRYBNnTpVhQsXlnThsNOgQYPUrFkzderUyeTqbgyFChdShYq3Ot/fWu4W3VWtkk6fStHvR4/p1KnTLv0dDodOnEjS/t8OSpL+TD2jxQuiNfzNgUpOPq3Tp1M0fNQAxe/aqw2XhF+PXh219adYnT1zTo3uq6ehb/xbo0dOVGqK++N0AG9ybP9G9idfld89LZS5Z5t8SpWXX/WGOv/139cyOrZ+pYBHesv36F5lHdkj3wp3yfe26kpf8tbfAxUKlq1wiGzFwiRJPqG3yDifduE6sbSz8ilVQT6lyyvz6G9S+lnZQkrKv34rZZ06oaw/9l/rzS7wbMZ1fLFVenq6295VbpQqVjkfqim46t37L0V/Pt+tfcmiFXrp+SFu7VvjvtX7M+a7XMhstwdo2KgBav34wwoMtGvT/1/I/Mfvic4+02aO0/0PNlLhwoX0274DmjFtjpYt+TR/NqqAO/C6+0NgcXk+FSMV0KDNhb2qlCRlbPtamTtcLx/xrVpf/ve0kK1IcRmnEpWx+VNl7o91Lvev96j86z3qNnb6qg+VufN72ULLKqBJB/mUDJf87TLOnlZmwk45fvxcxpnT+byFBUuhf8++Yp/rIsDWr1+vt99+W/Hx8bLZbKpcubIGDBjgfLyKpwgwFHQEGAq63ASY6ZNFH330kZo2bapChQqpb9++6tOnj4KCgnT//fdr0aJFZpcHALhOmb4HVrlyZT3zzDN65ZVXXNonTpyoWbNmKT4+3uMx2QNDQcceGAo6S+yBHThwQI888ohb+6OPPsqNfgEAOTI9wMLDw7Vmjfudz9esWaPw8HATKgIAWIFpp9H36NFDU6ZMUf/+/dW3b1/FxsaqXr16stls2rRpk+bOnaspU6aYVR4A4Dpn2hyYr6+vjh07prCwMK1YsULvvPOOc77r4lmIrVq1ytPYzIGhoGMODAVdbubATNsDuzQ3W7durdatW5tVCgDAgkydA7t4A00AADxl6q2k7rjjjiuGWHJy8jWqBgBgJaYG2MiRIxUSEmJmCQAAizI1wDp06KCwsDAzSwAAWJRpc2DMfwEAroZpAXYd3EMYAGBhph1CzMriMfIAgLwz/VZSAADkBQEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYks0wDMPsImBt6enpGjt2rAYPHiy73W52OYBX8f2+fhFguGqpqakKCQlRSkqKgoODzS4H8Cq+39cvDiECACyJAAMAWBIBBgCwJAIMV81ut2v48OFMcKNA4vt9/eIkDgCAJbEHBgCwJAIMAGBJBBgAwJIIMFyX5s6dq2LFipldBnBZBw8elM1mU2xsrNml3JAIsAKkW7dustlsGjdunEv7ypUrZbPZPBqrfPnymjx5co7Lv/vuO9lstsu+5s6dm4etuKB9+/bau3dvnj8PXHSl72m3bt3yPHZ4eLiOHTumqlWreq9g5Jqf2QXAuwIDAzV+/Hj17t1bxYsXz7f11KtXT8eOHXO+f+mll5Samqo5c+Y420JCQvI8flBQkIKCgq6qRkCSy/d0yZIlGjZsmPbs2eNsu5rvma+vr0qVKnVV9SHv2AMrYJo2bapSpUpp7Nixl+23fPly3XXXXbLb7Spfvrzeeecd57LGjRvr0KFDeuWVV5y/pf5TQECASpUq5XwFBQXJbrc73xcvXlyDBg1SWFiYAgMDde+992rr1q2SpLS0NN1111165plnnOMlJCQoJCREs2bNkpT9IcRPP/1UtWrVUmBgoEJDQ9WmTZu8/jHhBnLp9zQkJEQ2m82lbdGiRbrtttsUEBCgO++8UwsWLHB+tkePHoqMjFR6erokKSMjQ3fffbeeeuopSdkfQty5c6datmyp4OBgFS1aVA0aNND+/fuv6TbfKAiwAsbX11djxozRtGnTdPTo0Wz7bN++Xe3atVOHDh20Y8cOjRgxQkOHDnUe8ouOjtYtt9yiN954Q8eOHXP5DTa3Bg4cqOXLl2vevHn6+eefFRERoWbNmik5OVmBgYFauHCh5s2bp5UrVyozM1OdO3fWfffdp169emU73hdffKE2bdqoZcuWiomJ0Zo1a1SrVi2P6wIutWLFCr300kvq37+/fv31V/Xu3Vvdu3fXunXrJElTp07V2bNn9eqrr0qShg4dqqSkJE2fPj3b8X7//Xc1bNhQgYGBWrt2rbZv364ePXrI4XBcs226oRgoMLp27Wq0atXKMAzDqFOnjtGjRw/DMAxjxYoVxqX/qzt27Gg88MADLp8dMGCAUaVKFef7cuXKGZMmTcrTus+cOWP4+/sbCxcudC4/f/68UaZMGWPChAnOtgkTJhihoaHGiy++aJQqVco4efKkc9mcOXOMkJAQ5/u6desaTz31VK7rAbLzz+9VvXr1jF69ern0eeKJJ4wWLVo433///feGv7+/MXToUMPPz89Yv369c1lCQoIhyYiJiTEMwzAGDx5sVKhQwTh//ny+bgcuYA+sgBo/frzmzZunXbt2uS2Lj49X/fr1Xdrq16+vffv2KTMz86rXvX//fmVkZLisw9/fX7Vr11Z8fLyzrX///rrzzjs1bdo0zZkzR6GhoTmOGRsbq/vvv/+qawMuldPfhUu/p3Xr1tW///1vjRo1Sv3791fDhg1zHC82NlYNGjSQv79/vtWMvxFgBVTDhg3VrFkzDRkyxG2ZYRhu81qGF+8odnGs7NZxaduJEye0Z88e+fr6at++fZcdkxM6kF+u9D3NysrS5s2b+Z5ehwiwAmzcuHH67LPP9P3337u0V6lSRZs2bXJp+/7773XHHXfI19dX0oWTNPK6NxYREaGAgACXdWRkZGjbtm2qXLmys61Hjx6qWrWq5s+fr4EDB2a7t3hRZGSk1qxZk6d6gJxUrlw5278Ll35P33rrLcXHx2v9+vX66quvXM60/afIyEht3LhRGRkZ+VYzLmHqAUx41aXzUBd17tzZCAwMdJkD2759u+Hj42O88cYbxp49e4y5c+caQUFBxpw5c5x9HnjgAePRRx81jh496jI3ldt1v/TSS0aZMmWMVatWGTt37jS6du1qFC9e3EhOTjYMwzD+85//GMWKFTMOHz5sGIZhdOrUyYiKijLS09MNw3Cfq1i3bp3h4+NjDBs2zNi1a5cRFxdnjB8/3sM/Idzo/vm9WrFiheHv72/MmDHD2Lt3r/HOO+8Yvr6+xrp16wzDMIyYmBgjICDA+PTTTw3DMIzZs2cbRYsWNfbv328YhvscWFJSknHTTTcZbdq0MbZu3Wrs3bvXmD9/vrF79+5ruZk3DAKsAMkuwA4ePGjY7Xbjn7+rLFu2zKhSpYrh7+9v3HrrrcZbb73lsvyHH34wIiMjs/1sbtb9119/GS+++KIRGhpq2O12o379+sZPP/1kGIZhxMfHG0FBQcaiRYuc/VNSUozy5csbAwcONAzD/R8awzCM5cuXG1FRUUZAQIARGhpqtGnT5op1AZfK7ns1ffp0o2LFioa/v79xxx13GPPnzzcM48J3uEqVKsYzzzzj0r9169ZGvXr1DIfD4RZghmEYv/zyi/Hggw8ahQoVMooWLWo0aNDAGXjwLh6nAgCwJObAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsCAG0x2DwvNC5vNppUrV171OEBeEWBALlx8MnVOr27dul2zWrp166bHHnvsmq0PuF75mV0AYAWXPpV6yZIlGjZsmPbs2eNs++djNDIyMngmFJDP2AMDcqFUqVLOV0hIiGw2m/N9WlqaihUrpqVLl6px48YKDAzURx99pBEjRigqKsplnMmTJ6t8+fIubXPmzFHlypUVGBioSpUq5fi4+tyaOHGiqlWrpsKFCys8PFzPP/+8zpw549Zv5cqVuuOOOxQYGKgHHnhAR44ccVn+2Wef6e6771ZgYKAqVqyokSNHyuFwXFVtgDcRYICXDBo0SH379lV8fLyaNWuWq8/MmjVLr732mkaPHq34+HiNGTNGQ4cO1bx58/Jch4+Pj6ZOnapff/1V8+bN09q1azVw4ECXPufOndPo0aM1b948bd68WampqerQoYNz+VdffaVOnTqpb9++2rVrl9577z3NnTtXo0ePznNdgNeZfTt8wGr++UiOi4/UmDx5sku/4cOHG9WrV3dpmzRpklGuXDnn+/DwcJfHyhiGYYwaNcqoW7dujuvP7rE5l7N06VLjpptucqlfkvHjjz862+Lj4w1JxpYtWwzDMIwGDRoYY8aMcRlnwYIFRunSpZ3vJRkrVqzIdR2AtzEHBnhJrVq1POp/8uRJHTlyRE8//bR69erlbHc4HAoJCclzHevWrdOYMWO0a9cupaamyuFwKC0tTWfPnlXhwoUlSX5+fi71VqpUScWKFVN8fLxq166t7du3a+vWrS57XJmZmUpLS9O5c+dUqFChPNcHeAsBBnjJxXC4yMfHR8Y/Hrd36aPms7KyJF04jHjPPfe49PP19c1TDYcOHVKLFi307LPPatSoUSpRooQ2bdqkp59+2u0x9zabze3zF9uysrI0cuRItWnTxq1PYGBgnmoDvI0AA/JJyZIllZiYKMMwnMEQGxvrXH7zzTerbNmyOnDggJ566imvrHPbtm1yOBx655135ONzYYp76dKlbv0cDoe2bdum2rVrS5L27Nmj06dPq1KlSpKkmjVras+ePYqIiPBKXUB+IMCAfNK4cWOdPHlSEyZM0OOPP67Vq1dr1apVCg4OdvYZMWKE+vbtq+DgYD300ENKT0/Xtm3bdOrUKfXr1y/HsVNSUlzCUJJKlCih2267TQ6HQ9OmTdMjjzyizZs3a+bMmW6f9/f314svvqipU6fK399fffr0UZ06dZyBNmzYMD388MMKDw/XE088IR8fH8XFxWnHjh168803vfMHBFwtsyfhAKvJ6SSOmJgYt74zZswwwsPDjcKFCxtdunQxRo8e7XISh2EYxsKFC42oqCgjICDAKF68uNGwYUMjOjo6x/V37drVkOT26tq1q2EYhjFx4kSjdOnSRlBQkNGsWTNj/vz5hiTj1KlTLvUvX77cqFixohEQEGA0adLEOHjwoMt6Vq9ebdSrV88ICgoygoODjdq1axvvv/++c7k4iQMmsxnGPw7SAwBgAVwHBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkv4PU1PWcq4vlkEAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Print the confusion matrix.\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "mat = confusion_matrix(y_test, y_pred_class)\n",
        "sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False, \n",
        "            xticklabels=['Not Toxic', 'Toxic'], yticklabels=['Not Toxic', 'Toxic'])\n",
        "plt.xlabel('True Label')\n",
        "plt.ylabel('Predicted Label')\n",
        "\n",
        "metrics.confusion_matrix(y_test, y_pred_class)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "id": "Z7hZCuUQYFS7",
        "outputId": "2548849b-8254-4be2-92dc-81e869f6c188"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.90      0.86      0.88      9800\n",
            "           1       0.82      0.87      0.85      7570\n",
            "\n",
            "    accuracy                           0.86     17370\n",
            "   macro avg       0.86      0.86      0.86     17370\n",
            "weighted avg       0.87      0.86      0.86     17370\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Print the classification report.\n",
        "\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "print(classification_report(y_test, y_pred_class))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "id": "P01AYqS7YFS7",
        "outputId": "3023ce21-91e4-49f6-a625-cb944d81654f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([3.04449604e-05, 2.17034247e-07, 9.80302846e-09, ...,\n",
              "       1.69247459e-01, 9.99999602e-01, 9.99548799e-01])"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the predicted probabilities for the X_test_dtm (non-toxic class).\n",
        "\n",
        "y_pred_prob_non_toxic_class = mnb.predict_proba(X_test_dtm)[:, 0]\n",
        "y_pred_prob_non_toxic_class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "id": "dFHLhTPbhYiv"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([9.99969555e-01, 9.99999783e-01, 9.99999990e-01, ...,\n",
              "       8.30752541e-01, 3.98301173e-07, 4.51200757e-04])"
            ]
          },
          "execution_count": 55,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the predicted probabilities for the X_test_dtm (toxic class).\n",
        "\n",
        "y_pred_prob_toxic_class = mnb.predict_proba(X_test_dtm)[:, 1]\n",
        "y_pred_prob_toxic_class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "id": "zErA-Jr9YFS7",
        "outputId": "71abb37b-1673-40ca-fa45-bf53831591fb"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Di ako makapaniwala na pangalawa si Mar Roxas ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3.044496e-05</td>\n",
              "      <td>9.999696e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A lot of you niggas turned these bitches into ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.170342e-07</td>\n",
              "      <td>9.999998e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>RT @ZitlalyL: @Vic_Manzan0 just like you&amp;#1280...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>9.803028e-09</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>our playground for the morning.  big bear tri ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.955114e-01</td>\n",
              "      <td>4.488642e-03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"@ellisluciano: Binay as President #ShortHorro...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4.126098e-01</td>\n",
              "      <td>5.873902e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17365</th>\n",
              "      <td>Mad respect to Affleck for refusing to wear a ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.687911e-01</td>\n",
              "      <td>3.120894e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17366</th>\n",
              "      <td>RT @TAXSTONE: If YA bitch ex is a nigga u thin...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.133198e-14</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17367</th>\n",
              "      <td>@imcessfederico trillion yan ! Wala eh... Only...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1.692475e-01</td>\n",
              "      <td>8.307525e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17368</th>\n",
              "      <td>fathersdaymessage  #fathers #day #message buy...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.999996e-01</td>\n",
              "      <td>3.983012e-07</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17369</th>\n",
              "      <td>@user my woodpecker shot in this months issue...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.995488e-01</td>\n",
              "      <td>4.512008e-04</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>17370 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "0      Di ako makapaniwala na pangalawa si Mar Roxas ...       1          1   \n",
              "1      A lot of you niggas turned these bitches into ...       1          1   \n",
              "2      RT @ZitlalyL: @Vic_Manzan0 just like you&#1280...       1          1   \n",
              "3      our playground for the morning.  big bear tri ...       0          0   \n",
              "4      \"@ellisluciano: Binay as President #ShortHorro...       1          1   \n",
              "...                                                  ...     ...        ...   \n",
              "17365  Mad respect to Affleck for refusing to wear a ...       0          0   \n",
              "17366  RT @TAXSTONE: If YA bitch ex is a nigga u thin...       1          1   \n",
              "17367  @imcessfederico trillion yan ! Wala eh... Only...       0          1   \n",
              "17368   fathersdaymessage  #fathers #day #message buy...       0          0   \n",
              "17369   @user my woodpecker shot in this months issue...       0          0   \n",
              "\n",
              "       prob_non_toxic    prob_toxic  \n",
              "0        3.044496e-05  9.999696e-01  \n",
              "1        2.170342e-07  9.999998e-01  \n",
              "2        9.803028e-09  1.000000e+00  \n",
              "3        9.955114e-01  4.488642e-03  \n",
              "4        4.126098e-01  5.873902e-01  \n",
              "...               ...           ...  \n",
              "17365    9.687911e-01  3.120894e-02  \n",
              "17366    1.133198e-14  1.000000e+00  \n",
              "17367    1.692475e-01  8.307525e-01  \n",
              "17368    9.999996e-01  3.983012e-07  \n",
              "17369    9.995488e-01  4.512008e-04  \n",
              "\n",
              "[17370 rows x 5 columns]"
            ]
          },
          "execution_count": 56,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Build a data frame with the input text, actual class, predicted class, and the predicted probabilities.\n",
        "\n",
        "test_results_df = pd.DataFrame({'text': X_test, 'actual': y_test, 'predicted': y_pred_class, 'prob_non_toxic': y_pred_prob_non_toxic_class, 'prob_toxic': y_pred_prob_toxic_class})\n",
        "\n",
        "# Reset the index.\n",
        "test_results_df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "test_results_df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "6zSvXrzgYFS7"
      },
      "source": [
        "# 3 | Hypertuning the parameters."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "HeNzh4HVmVCy"
      },
      "source": [
        "Since our dataset also contains Filipino and TagLish entries, we decided to make a custom stopwords list that include these languages."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "TyyqW7C1YFS8"
      },
      "source": [
        "## Making a custom stop words."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {
        "id": "OyBUhkfoYFS8"
      },
      "outputs": [],
      "source": [
        "# Making a custom stop words list that includes the Filipino language.\n",
        "\n",
        "# Built-in English stop words.\n",
        "english_stop_words = CountVectorizer(stop_words='english').get_stop_words()\n",
        "\n",
        "# Generated by ChatGPT.\n",
        "filipino_stop_words1 = [\n",
        "    \"ako\", \"akin\", \"ako'y\", \"amin\", \"aming\", \"ang\", \"ano\", \"anuman\", \"apat\", \"at\", \"atin\", \"ating\",\n",
        "    \"ay\", \"bababa\", \"bago\", \"bakit\", \"bawat\", \"bilang\", \"dahil\", \"dalawa\", \"dapat\", \"din\", \"dito\", \"doon\",\n",
        "    \"gagawin\", \"gayunman\", \"ginagawa\", \"ginawa\", \"ginawang\", \"gumawa\", \"gusto\", \"habang\", \"hanggang\", \"hindi\", \"huwag\", \"iba\",\n",
        "    \"ibaba\", \"ibabaw\", \"ibig\", \"ikaw\", \"ilagay\", \"ilalim\", \"ilan\", \"inyong\", \"isa\", \"isang\", \"ito\", \"iyo\",\n",
        "    \"iyon\", \"iyong\", \"ka\", \"kahit\", \"kailangan\", \"kailanman\", \"kami\", \"kanila\", \"kanilang\", \"kanino\", \"kanya\", \"kanyang\",\n",
        "    \"kapag\", \"kapwa\", \"karamihan\", \"katiyakan\", \"katulad\", \"kay\", \"kaya\", \"kaysa\", \"ko\", \"kung\", \"laban\",\n",
        "    \"lahat\", \"lamang\", \"likod\", \"lima\", \"maaari\", \"maaaring\", \"maging\", \"mahusay\", \"makita\", \"marami\", \"marapat\", \"mga\",\n",
        "    \"minsan\", \"mismo\", \"mula\", \"muli\", \"na\", \"nabanggit\", \"naging\", \"nagkaroon\", \"nais\", \"nakita\", \"namin\", \"napaka\",\n",
        "    \"narito\", \"nasaan\", \"ng\", \"nga\", \"ngayon\", \"ni\", \"nila\", \"nilang\", \"nito\", \"niya\", \"niyang\", \"noon\",\n",
        "    \"o\", \"pag\", \"pala\", \"para\", \"pati\", \"pero\", \"pumunta\", \"pumupunta\", \"sa\", \"saan\", \"sabi\", \"sabihin\",\n",
        "    \"sarili\", \"si\", \"sila\", \"sino\", \"siya\", \"tatlo\", \"tayo\", \"tulad\", \"tungkol\", \"una\", \"walang\"\n",
        "]\n",
        "\n",
        "# From an open-source GitHub repo (ref: https://github.com/stopwords-iso/stopwords-tl).\n",
        "filipino_stop_words2 = [\n",
        "    'ako', 'sa', 'akin', 'ko', 'aking', 'sarili', 'kami', 'atin', 'ang', 'aming', 'amin', 'ating',\n",
        "    'ka', 'iyong', 'iyo', 'inyong', 'siya', 'kanya', 'mismo', 'ito', 'nito', 'kanyang', 'sila', 'nila',\n",
        "    'kanila', 'kanilang', 'kung', 'ano', 'alin', 'sino', 'kanino', 'na', 'mga', 'iyon', 'am', 'ay',\n",
        "    'maging', 'naging', 'mayroon', 'may', 'nagkaroon', 'pagkakaroon', 'gumawa', 'ginagawa', 'ginawa', 'paggawa',\n",
        "    'ibig', 'dapat', 'maaari', 'marapat', 'kong', 'ikaw', 'tayo', 'hindi', 'namin', 'gusto', 'nais',\n",
        "    'niyang', 'nilang', 'niya', 'huwag', 'ginawang', 'gagawin', 'maaaring', 'sabihin', 'narito', 'kapag', 'ni',\n",
        "    'nasaan', 'bakit', 'paano', 'kailangan', 'walang', 'katiyakan', 'isang', 'at', 'pero', 'o', 'dahil',\n",
        "    'bilang', 'hanggang', 'habang', 'ng', 'pamamagitan', 'para', 'tungkol', 'laban', 'pagitan', 'panahon', 'bago',\n",
        "    'pagkatapos', 'itaas', 'ibaba', 'mula', 'pataas', 'pababa', 'palabas', 'ibabaw', 'ilalim', 'muli', 'pa',\n",
        "    'minsan', 'dito', 'doon', 'saan', 'lahat', 'anumang', 'kapwa', 'bawat', 'ilan', 'karamihan', 'iba', 'tulad',\n",
        "    'lamang', 'pareho', 'kaya', 'kaysa', 'masyado', 'napaka', 'isa', 'bababa', 'kulang', 'marami', 'ngayon',\n",
        "    'kailanman', 'sabi', 'nabanggit', 'din', 'kumuha', 'pumunta', 'pumupunta', 'ilagay', 'makita', 'nakita',\n",
        "    'katulad', 'mahusay', 'likod', 'kahit', 'paraan', 'noon', 'gayunman', 'dalawa', 'tatlo', 'apat', 'lima',\n",
        "    'una', 'pangalawa'\n",
        "]\n",
        "\n",
        "# Get the union of the two filipino stop words lists.\n",
        "filipino_stop_words = list(set(filipino_stop_words1).union(set(filipino_stop_words2)))\n",
        "\n",
        "# Get the union of the english and filipino stop words lists.\n",
        "custom_stop_words = list(english_stop_words.union(filipino_stop_words))\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "1z_llSMBYFS8"
      },
      "source": [
        "## Search for the optimal parameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zr-B9-RkYFS8",
        "outputId": "44610b39-796b-4ab9-fdc1-422dc3b3b311"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fitting 5 folds for each of 192 candidates, totalling 960 fits\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "{'cvec__max_df': 0.2,\n",
              " 'cvec__min_df': 2,\n",
              " 'cvec__ngram_range': (1, 2),\n",
              " 'cvec__stop_words': ['sabi',\n",
              "  'pumunta',\n",
              "  'however',\n",
              "  'yours',\n",
              "  'down',\n",
              "  'una',\n",
              "  'due',\n",
              "  'meanwhile',\n",
              "  'together',\n",
              "  'with',\n",
              "  'after',\n",
              "  'sarili',\n",
              "  'kapwa',\n",
              "  'had',\n",
              "  'anything',\n",
              "  'back',\n",
              "  'he',\n",
              "  'itaas',\n",
              "  'eleven',\n",
              "  'as',\n",
              "  'against',\n",
              "  'four',\n",
              "  'much',\n",
              "  'per',\n",
              "  'someone',\n",
              "  'whole',\n",
              "  'along',\n",
              "  'but',\n",
              "  'around',\n",
              "  'six',\n",
              "  'hindi',\n",
              "  'kahit',\n",
              "  'panahon',\n",
              "  'tulad',\n",
              "  'kanilang',\n",
              "  'maaaring',\n",
              "  'latter',\n",
              "  'ginawang',\n",
              "  'now',\n",
              "  'perhaps',\n",
              "  'she',\n",
              "  'ilan',\n",
              "  'nothing',\n",
              "  'indeed',\n",
              "  'thence',\n",
              "  'not',\n",
              "  'front',\n",
              "  'through',\n",
              "  'many',\n",
              "  'sino',\n",
              "  'gayunman',\n",
              "  'we',\n",
              "  'ako',\n",
              "  'sila',\n",
              "  'detail',\n",
              "  'saan',\n",
              "  'bawat',\n",
              "  'do',\n",
              "  'therefore',\n",
              "  'interest',\n",
              "  'move',\n",
              "  'amount',\n",
              "  'towards',\n",
              "  'whither',\n",
              "  'where',\n",
              "  'you',\n",
              "  'noone',\n",
              "  'atin',\n",
              "  'thereby',\n",
              "  'isang',\n",
              "  'para',\n",
              "  'whence',\n",
              "  'cry',\n",
              "  'his',\n",
              "  'whenever',\n",
              "  'fifteen',\n",
              "  'hanggang',\n",
              "  'seeming',\n",
              "  'at',\n",
              "  'kung',\n",
              "  'marami',\n",
              "  'while',\n",
              "  'or',\n",
              "  'everyone',\n",
              "  'noon',\n",
              "  'then',\n",
              "  'within',\n",
              "  'sincere',\n",
              "  're',\n",
              "  'thereupon',\n",
              "  'habang',\n",
              "  'anyone',\n",
              "  'latterly',\n",
              "  'pagkatapos',\n",
              "  'own',\n",
              "  'is',\n",
              "  'ibig',\n",
              "  'might',\n",
              "  'ni',\n",
              "  'from',\n",
              "  'further',\n",
              "  'nais',\n",
              "  'empty',\n",
              "  'became',\n",
              "  'why',\n",
              "  'pataas',\n",
              "  'any',\n",
              "  'bilang',\n",
              "  'kanino',\n",
              "  'kaysa',\n",
              "  'con',\n",
              "  'none',\n",
              "  'ours',\n",
              "  'when',\n",
              "  'eight',\n",
              "  'apat',\n",
              "  'mula',\n",
              "  'whatever',\n",
              "  'inc',\n",
              "  'put',\n",
              "  'him',\n",
              "  'whereupon',\n",
              "  'by',\n",
              "  'laban',\n",
              "  'describe',\n",
              "  'found',\n",
              "  'also',\n",
              "  'nevertheless',\n",
              "  'those',\n",
              "  'whereafter',\n",
              "  'gusto',\n",
              "  'are',\n",
              "  'anumang',\n",
              "  'for',\n",
              "  'paraan',\n",
              "  'between',\n",
              "  'herself',\n",
              "  'again',\n",
              "  'other',\n",
              "  'beside',\n",
              "  'kumuha',\n",
              "  'kong',\n",
              "  'alin',\n",
              "  'wherever',\n",
              "  'made',\n",
              "  'they',\n",
              "  'will',\n",
              "  'three',\n",
              "  'sometimes',\n",
              "  'aking',\n",
              "  'rather',\n",
              "  'same',\n",
              "  'few',\n",
              "  'there',\n",
              "  'should',\n",
              "  'without',\n",
              "  'un',\n",
              "  'almost',\n",
              "  'more',\n",
              "  'toward',\n",
              "  'mga',\n",
              "  'afterwards',\n",
              "  'thereafter',\n",
              "  'paano',\n",
              "  'nasaan',\n",
              "  'ano',\n",
              "  'others',\n",
              "  'iyo',\n",
              "  'ngayon',\n",
              "  'twenty',\n",
              "  'co',\n",
              "  'twelve',\n",
              "  'kami',\n",
              "  'itself',\n",
              "  'de',\n",
              "  'her',\n",
              "  'here',\n",
              "  'until',\n",
              "  'on',\n",
              "  'give',\n",
              "  'else',\n",
              "  'already',\n",
              "  'amoungst',\n",
              "  'niya',\n",
              "  'whoever',\n",
              "  'if',\n",
              "  'me',\n",
              "  'take',\n",
              "  'these',\n",
              "  'yourselves',\n",
              "  'kulang',\n",
              "  'pagitan',\n",
              "  'hasnt',\n",
              "  'amongst',\n",
              "  'them',\n",
              "  'would',\n",
              "  'amin',\n",
              "  'behind',\n",
              "  'been',\n",
              "  'show',\n",
              "  'ever',\n",
              "  'serious',\n",
              "  'couldnt',\n",
              "  'because',\n",
              "  'nakita',\n",
              "  'how',\n",
              "  'next',\n",
              "  'anywhere',\n",
              "  'himself',\n",
              "  'hereupon',\n",
              "  'ginagawa',\n",
              "  'iyong',\n",
              "  \"ako'y\",\n",
              "  'inyong',\n",
              "  'beyond',\n",
              "  'being',\n",
              "  'can',\n",
              "  'lamang',\n",
              "  'always',\n",
              "  'everywhere',\n",
              "  'fifty',\n",
              "  'tayo',\n",
              "  'bill',\n",
              "  'whereby',\n",
              "  'who',\n",
              "  'i',\n",
              "  'bago',\n",
              "  'former',\n",
              "  'forty',\n",
              "  'the',\n",
              "  'what',\n",
              "  'thin',\n",
              "  'otherwise',\n",
              "  'walang',\n",
              "  'an',\n",
              "  'part',\n",
              "  'some',\n",
              "  'karamihan',\n",
              "  'every',\n",
              "  'of',\n",
              "  'upon',\n",
              "  'pababa',\n",
              "  'off',\n",
              "  'ten',\n",
              "  'beforehand',\n",
              "  'kapag',\n",
              "  'anuman',\n",
              "  'several',\n",
              "  'pagkakaroon',\n",
              "  'pero',\n",
              "  'namin',\n",
              "  'pumupunta',\n",
              "  'side',\n",
              "  'very',\n",
              "  'only',\n",
              "  'thick',\n",
              "  'ang',\n",
              "  'gumawa',\n",
              "  'hereafter',\n",
              "  'dahil',\n",
              "  'masyado',\n",
              "  'herein',\n",
              "  'marapat',\n",
              "  'kanya',\n",
              "  'etc',\n",
              "  'last',\n",
              "  'na',\n",
              "  'during',\n",
              "  'least',\n",
              "  'your',\n",
              "  'name',\n",
              "  'go',\n",
              "  'paggawa',\n",
              "  'besides',\n",
              "  'though',\n",
              "  'could',\n",
              "  'over',\n",
              "  'minsan',\n",
              "  'and',\n",
              "  'than',\n",
              "  'pala',\n",
              "  'were',\n",
              "  'maaari',\n",
              "  'ibabaw',\n",
              "  'siya',\n",
              "  'become',\n",
              "  'ito',\n",
              "  'makita',\n",
              "  'nabanggit',\n",
              "  'nor',\n",
              "  'formerly',\n",
              "  'hundred',\n",
              "  'aming',\n",
              "  'napaka',\n",
              "  'katiyakan',\n",
              "  'nagkaroon',\n",
              "  'myself',\n",
              "  'no',\n",
              "  'moreover',\n",
              "  'ka',\n",
              "  'katulad',\n",
              "  'see',\n",
              "  'bakit',\n",
              "  'lima',\n",
              "  'our',\n",
              "  'am',\n",
              "  'ibaba',\n",
              "  'has',\n",
              "  'kailangan',\n",
              "  'narito',\n",
              "  'was',\n",
              "  'pati',\n",
              "  'five',\n",
              "  'mayroon',\n",
              "  'must',\n",
              "  'eg',\n",
              "  'ikaw',\n",
              "  'dito',\n",
              "  'its',\n",
              "  'kaya',\n",
              "  'ie',\n",
              "  'each',\n",
              "  'yet',\n",
              "  'in',\n",
              "  'sometime',\n",
              "  'anyway',\n",
              "  'tungkol',\n",
              "  'ating',\n",
              "  'lahat',\n",
              "  'my',\n",
              "  'before',\n",
              "  'once',\n",
              "  'under',\n",
              "  'please',\n",
              "  'seemed',\n",
              "  'up',\n",
              "  'never',\n",
              "  'ourselves',\n",
              "  'most',\n",
              "  'full',\n",
              "  'have',\n",
              "  'likod',\n",
              "  'mismo',\n",
              "  'muli',\n",
              "  'mahusay',\n",
              "  'doon',\n",
              "  'hers',\n",
              "  'therein',\n",
              "  'mine',\n",
              "  'pag',\n",
              "  'find',\n",
              "  'fire',\n",
              "  'may',\n",
              "  'which',\n",
              "  'dapat',\n",
              "  'still',\n",
              "  'huwag',\n",
              "  'it',\n",
              "  'so',\n",
              "  'nobody',\n",
              "  'via',\n",
              "  'nila',\n",
              "  'din',\n",
              "  'either',\n",
              "  'top',\n",
              "  'something',\n",
              "  'well',\n",
              "  'above',\n",
              "  'bababa',\n",
              "  'ay',\n",
              "  'yourself',\n",
              "  'whether',\n",
              "  'sa',\n",
              "  'whereas',\n",
              "  'wherein',\n",
              "  'cant',\n",
              "  'ng',\n",
              "  'kanyang',\n",
              "  'first',\n",
              "  'kailanman',\n",
              "  'onto',\n",
              "  'ilagay',\n",
              "  'gagawin',\n",
              "  'that',\n",
              "  'us',\n",
              "  'becoming',\n",
              "  'even',\n",
              "  'among',\n",
              "  'two',\n",
              "  'naging',\n",
              "  'thru',\n",
              "  'into',\n",
              "  'o',\n",
              "  'somewhere',\n",
              "  'be',\n",
              "  'their',\n",
              "  'sixty',\n",
              "  'alone',\n",
              "  'hence',\n",
              "  'nowhere',\n",
              "  'mostly',\n",
              "  'fill',\n",
              "  'becomes',\n",
              "  'seems',\n",
              "  'bottom',\n",
              "  'about',\n",
              "  'elsewhere',\n",
              "  'iba',\n",
              "  'ltd',\n",
              "  'maging',\n",
              "  'iyon',\n",
              "  'nine',\n",
              "  'cannot',\n",
              "  'done',\n",
              "  'system',\n",
              "  'niyang',\n",
              "  'akin',\n",
              "  'ginawa',\n",
              "  'somehow',\n",
              "  'tatlo',\n",
              "  'both',\n",
              "  'sabihin',\n",
              "  'ko',\n",
              "  'dalawa',\n",
              "  'too',\n",
              "  'themselves',\n",
              "  'although',\n",
              "  'kanila',\n",
              "  'keep',\n",
              "  'nito',\n",
              "  'out',\n",
              "  'si',\n",
              "  'to',\n",
              "  'this',\n",
              "  'everything',\n",
              "  'nga',\n",
              "  'one',\n",
              "  'enough',\n",
              "  'pa',\n",
              "  'whom',\n",
              "  'neither',\n",
              "  'below',\n",
              "  'another',\n",
              "  'less',\n",
              "  'hereby',\n",
              "  'isa',\n",
              "  'namely',\n",
              "  'thus',\n",
              "  'pareho',\n",
              "  'get',\n",
              "  'since',\n",
              "  'seem',\n",
              "  'ilalim',\n",
              "  'pamamagitan',\n",
              "  'nilang',\n",
              "  'kay',\n",
              "  'throughout',\n",
              "  'a',\n",
              "  'except',\n",
              "  'whose',\n",
              "  'anyhow',\n",
              "  'such',\n",
              "  'third',\n",
              "  'often',\n",
              "  'call',\n",
              "  'across',\n",
              "  'palabas',\n",
              "  'pangalawa',\n",
              "  'all',\n",
              "  'mill'],\n",
              " 'mnb__alpha': 1.0}"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Hyperparameter tuning using GridSearchCV.\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Create a pipeline with CountVectorizer and MultinomialNB.\n",
        "pipe = Pipeline([\n",
        "    ('cvec', CountVectorizer()),\n",
        "    ('mnb', MultinomialNB())\n",
        "])\n",
        "\n",
        "# Create a dictionary of parameters to search.\n",
        "params = {\n",
        "    'cvec__stop_words': [custom_stop_words],\n",
        "    'cvec__ngram_range': [(1, 1), (1, 2), (1, 3)],\n",
        "    'cvec__min_df': [1, 2, 3, 4],\n",
        "    'cvec__max_df': [0.01, 0.1, 0.2, 0.3],\n",
        "    'mnb__alpha': [0.01, 0.1, 1.0, 10.0],\n",
        "}\n",
        "\n",
        "# Instantiate GridSearchCV.\n",
        "gs = GridSearchCV(pipe, param_grid=params, cv=5, verbose=2, n_jobs=-1, scoring='accuracy', refit=True)\n",
        "\n",
        "# Fit GridSearch to training data.\n",
        "gs.fit(X_train, y_train)\n",
        "gs.best_params_"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "CwXRUzJ2YFS8"
      },
      "source": [
        "# 4 | Using the new parameters."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "isX_lNYRYFS8"
      },
      "source": [
        "## Building the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "id": "-b2JcjCHYFS8",
        "outputId": "29ad5ff4-17db-44f5-de75-dd2759379915"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MultinomialNB()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div>"
            ],
            "text/plain": [
              "MultinomialNB()"
            ]
          },
          "execution_count": 58,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Build a new model using the best parameters.\n",
        "\n",
        "cvect = CountVectorizer(stop_words=custom_stop_words, ngram_range=(1, 2), max_df=0.2, min_df=2, lowercase=False, binary=True)\n",
        "mnb = MultinomialNB(alpha=1.0)\n",
        "\n",
        "# Vectorize the data.\n",
        "X_train_dtm = cvect.fit_transform(X_train)\n",
        "X_test_dtm = cvect.transform(X_test)\n",
        "\n",
        "# Fit the data to the model.\n",
        "mnb.fit(X_train_dtm, y_train)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "lbtVeKtjYFS8"
      },
      "source": [
        "## Evaluating the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "id": "fGsH_ijqYFS8"
      },
      "outputs": [],
      "source": [
        "# Make class predictions for the X_test_dtm. Store it in the y_pred_class variable.\n",
        "y_pred_class = mnb.predict(X_test_dtm)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "id": "yryK5FD3YFS9",
        "outputId": "3099cf0c-b3c0-40bf-c05d-34b041fb3f62"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.8709268854346575"
            ]
          },
          "execution_count": 60,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "\n",
        "metrics.accuracy_score(y_test, y_pred_class)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "id": "WhyDUgf8YFS9",
        "outputId": "327808be-7c12-44b8-bfed-532be15787ab"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[8495, 1305],\n",
              "       [ 937, 6633]], dtype=int64)"
            ]
          },
          "execution_count": 61,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAGwCAYAAADITjAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyzklEQVR4nO3de3zP9f//8ft7p/fmsBmzHNLCSsSM5IPlkBRRCZGPM9FBRQipiOSYHAtFOYVSDp9OOnwlx5LD1sQcYiLModmWw2bv7fX7w8+bd9vYe97z8prb9XLZ5bP38/V8P9+PV5933fd6PZ+v18tmGIYhAAAsxsvsAgAAyAsCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWJKP2QXkh/ST+80uAchXAWXqm10CkK8c5w9ftQ9HYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkkwPsCeeeEJjx47N0v7222+rbdu2JlQEALAC0wNszZo1atGiRZb2Zs2aae3atSZUBACwAtMD7PTp0/Lz88vS7uvrq5SUFBMqAgBYgekBVrVqVX366adZ2j/55BNVqVLFhIoAAFbgY3YBQ4cOVZs2bbRv3z41btxYkrRq1SotXrxYn332mcnVAQBuVKYH2GOPPaYVK1Zo9OjR+vzzzxUQEKCIiAj93//9nxo2bGh2eQCAG5TNMAzD7CI8Lf3kfrNLAPJVQJn6ZpcA5CvH+cNX7WP6HBgAAHlhyinE4sWLa8+ePQoJCVFwcLBsNluOfRMTE69jZQAAqzAlwCZNmqSiRYs6f79SgAEAkB3mwAALYg4MBZ0l5sA+/PDDbNsdDoeGDBlynasBAFiF6QE2YMAAtWnTxmWua9euXapdu7aWLFliYmUAgBuZ6QEWHR2tY8eOqVq1avrhhx/03nvvqWbNmqpatapiYmLMLg8AcIMy/ULm8uXLa+3aterXr5+aNWsmb29vzZ8/X+3btze7NADADcz0IzBJ+uqrr7R48WLVq1dPxYoV06xZs3TkyBGzywIA3MBMD7BnnnlG7dq106BBg7R27VrFxsbKbrerWrVqzIEBAHJk+jL6qlWrauHChapevbpL+3vvvafBgwfr9OnTbo/JMnoUdCyjR0GXm2X0pgdYWlqa7HZ7ttt2796tSpUquT0mAYaCjgBDQZebADN9EYfdbldSUpI+/PBDxcXFyWazqXLlynrqqafyFF4AgJuD6UdgW7ZsUdOmTRUQEKDatWvLMAxt2bJF586d0/fff6+aNWu6PSZHYCjoOAJDQWeJU4j169dXeHi4Zs2aJR+fCweEDodDPXv21P79+7V27Vq3xyTAUNARYCjoLBFgAQEBio6O1l133eXSvnPnTtWqVUtnz551e0wCDAUdAYaCzhL3QgwMDNTBgweztB86dMh5x3rkL4cjQ1M/mKemT3TTPfe3VLO23TXjo4XKzMzMtv+I8VNVNephLfh0uUv7wb+OqM+QN1W/xZP6z4OtNWDoaJ1MPOXS56E2XVU16mGXn0kzPsq3fQNyUqRIYb0zYYT27d2kf5L/0Lo1/1Otey6thh42tL9+375Gyaf26sSxHfpu5SeqfW8N5/awsFvlOH842582bR4xY5duOqYt4pg/f76efPJJPfnkk3rqqac0YcIE1atXTzabTevXr9fAgQP13//+16zybiofLlyiJSu+0ajXByi8fJh27Nqj10dNUpEihdW53eMufVet3ajYHbsVGlLCpf3suVQ93e81VQqvoA+njpUkvTtrgV4YNFyLPpgkL69Lfyu90LOznnismfN1oYCA/Ns5IAcfvD9Bd99dSd2699GRo8fUsUNrffftJ6pW/X4dOZKgPXv3q2/f17U//k8FBPirb59eWvnNIlWqHKWTJxN16NARlS0X6TJmr54d9fKA3vr22x/N2ambjGkB1r17dzVr1kwTJkyQzWZTly5d5HA4JEm+vr567rnnNHbsWLPKu6n89vsu3V+/jhrWqy1JKlv6Fn3zwxrt2LXXpd+xEyc1euJ0vT9xlHoPHOayLTp2h44kHNfnc99VkcKFJUkjX+2nqIfbadPW31T3sr9cCxcKUEiJ4vm8V0DO/P391bpVc7Vu00Pr1m+SJL05cqIee6yZnn2mi4a9MV6ffLLC5T0vDxyhp3p0UES1Kvpx9XplZmbq2LETLn1atnxYSz77QmfOuD/1AfeZdgrx4tSbn5+fpkyZolOnTikmJkbR0dFKTEzUpEmTcrw+DJ5VM+JubdoSowMH/5Ik7dq7X9tid6hB3XudfTIzMzXkzQnq1uEJhVcIyzJGenq6bDbJz9fX2Wa3+8nLy0vbYne49P1w4WeKerid2nR9Xu/PW6z09PR82jMgez4+3vLx8VFqappLe+q5VEXVuzdLf19fX/Xq2VFJScn67V/f54tq1qimGpFVNWfOJ/lSM7Iy9Tqwy5/EXKhQIVWrVs3Eam5eT3Vqq39On9GjHZ6Wt5eXMjIz1efprmr+YCNnnw8//kze3l7q1LZltmNE3H2XAvz9NXH6R+r7bDcZhjRp+kfKzMzUyb8vPSqnU9vHVblSRQUWLartO3dryvtzdPjIMb055KV83kvgktOnz+jnn7fotVf7Km7XXh07dkLt2z+u2rVraO8f8c5+LZo30cKPp6tQoQAdPXpMzR7+r/7++1S2Y3bv/l/tjNujn3/Zcr1246ZnaoB169btqkdZy5Ytu+L2tLQ0paW5/hXldYW7eyCrlavW6Kvvf9S44YMUXj5Mu/bu17gp7ys0pLhaNn9QO3bt1cef/U+ffTTN5Y+OyxUPLqZ3Rr6qkRPe1cLPv5CXl00PN2mkKpXCXea/urRv5fy9Unh5BRUton6vj1L/3j1ULCgw3/cVuKhr9z6a/cE7OvTnNjkcDkVHb9fiT5arRo1Lf0iv/mmD7rn3IYWUKK6nnuqgxYtmqt59j+jEib9dxvL399d/2z+uUaOnXO/duKmZGmBFixZVwDVO4I8ZM0YjRoxwaXt9YB8NG9T3msa9mbzz3ofq2amdmjdpJEm6s2J5HU04rtkLlqhl8we17bfflXgqSQ+26eJ8T0ZGpt5+d7YWLFmh75fOkyRF/eceffvZHJ1KSpa3t7cCixZRw0c7qFnpUjl+dkTVC5dPHPzrCAGG62r//j/VuMkTKlQoQIGBRZWQcFyLFs7QgfhDzj5nz57Tvn0HtG/fAW36dZvidqxXj+7/1bjx77qM1aZNCxUqFKAFH392vXfjpmZqgE2dOlWhoaHXNMaQIUPUv39/lzavf65+/QAuSU1Nk83L9cjKy8tLmf9/nvLRZg+ozmWLMCTpmX6v69FmjfV484eyjBdcLEiStGlrjBJPJen+++rk+Nlxe/ZJkkqyqAMmOXv2nM6ePadixYL00IMN9cqQUTn2tdkuzO3+W49u7fXlVz/o5MnEbN6F/GJagOV0Kspddrs9y+nC9PMnPTL2zaJR1H80a94nKn1LqMLLhyluzx+a/+kytWpxIZyKBQVmOTry8fFWSPFglQ+71dm2/OvvVSGsnIKLBem3Hbs0dvJMdXmylbNPzO9xiv19l2rXjFCRIoX1e9wejZ/6ge6/r45Kl7q2P2QAdz30YEPZbDbt3rNP4RVv19ixQ7Vnzz7NnfepChUK0KtD+urLL7/X0YRjKlE8WM8+21W33lpany/9ymWcihVvV/36dfToY51N2pObl2kBZvINQHCZV/s9p2mz5uutCe8p8VSSSoYUV9uWzfVc9w5ujXPg4F+aPHOuklP+UdnSt+jpru3V5clLc15+vr76dtUazZizUOfPp6tMqVC1eayZenR8wtO7BFxVYFCgRo18RbfeWlqJiUlatvwbDR02Tg6HQ97e3qpUqaI6d/pAISHF9fffp7Rl629qdH9r7dy5x2Wc7t3a6/DhBH3/wxqT9uTmZdqtpNasWaOoqCjn/Q89iVtJoaDjVlIo6G7ox6k0bNjQrI8GABQApt8LEQCAvCDAAACWRIABACzJ9ADz9vbW8ePHs7T//fff8vb2NqEiAIAVmB5gOS2CTEtLk59f1gsGAQCQTFyFOHXqVEkXLmiePXu2ihQp4tyWkZGhtWvXZnlKMwAAF5kWYJMmTZJ04Qhs5syZLqcL/fz8dPvtt2vmzJlmlQcAuMGZFmDx8RceWXD//fdr2bJlCg4ONqsUAIAFmXozX0lavXq18/eL82Geuk8iAKDgMn0RhyTNnz9f1apVU0BAgAICAhQREaEFCxaYXRYA4AZm+hHYxIkTNXToUL3wwguKioqSYRjasGGDnn32WZ08eVL9+vUzu0QAwA3ItJv5XlS+fHmNGDFCXbp0cWmfN2+ehg8f7pwrcwc380VBx818UdDl5ma+pp9CPHr0qOrVq5elvV69ejp69KgJFQEArMD0AAsPD9eSJUuytH/66ae64447TKgIAGAFps+BjRgxQk8++aTWrl2rqKgo2Ww2rV+/XqtWrco22AAAkG6AI7A2bdpo06ZNCgkJ0YoVK7Rs2TKFhITo119/VatWra4+AADgpmT6Io78wCIOFHQs4kBBZ4lFHAAA5IVpc2BeXl5XveOGzWaTw+G4ThUBAKzEtABbvnx5jts2btyoadOm5fioFQAATAuwli1bZmnbtWuXhgwZoi+//FIdO3bUyJEjTagMAGAFN8Qc2JEjR9SrVy9FRETI4XAoJiZG8+bN02233WZ2aQCAG5SpAZacnKzBgwcrPDxcO3bs0KpVq/Tll1+qatWqZpYFALAA004hjh8/XuPGjVOpUqW0ePHibE8pAgCQE9OuA/Py8lJAQICaNGni8jTmf1u2bJnbY3MdGAo6rgNDQZeb68BMOwLr0qULD64EAOQZd+IALIgjMBR03IkDAFBgEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCScnUnjtjY2FwPGBERkediAADIrVwFWGRkpGw2W44PmLy4zWazKSMjw6MFAgCQnVwFWHx8fH7XAQCAW3IVYGFhYfldBwAAbsnTIo4FCxYoKipKZcqU0Z9//ilJmjx5sv73v/95tDgAAHLidoDNmDFD/fv3V/PmzZWUlOSc8ypWrJgmT57s6foAAMiW2wE2bdo0zZo1S6+99prLgyhr1aql7du3e7Q4AABy4naAxcfHq0aNGlna7Xa7zpw545GiAAC4GrcDrHz58oqJicnSvnLlSlWpUsUTNQEAcFW5WoV4uYEDB+r5559XamqqDMPQr7/+qsWLF2vMmDGaPXt2ftQIAEAWbgdY9+7d5XA4NGjQIJ09e1YdOnRQ2bJlNWXKFLVv3z4/agQAIAubkdPtNXLh5MmTyszMVGhoqCdrumbpJ/ebXQKQrwLK1De7BCBfOc4fvmoft4/ALjp+/Lh2794tm80mm82mkiVL5nUoAADc5vYijpSUFHXu3FllypRRw4YN1aBBA5UpU0adOnVScnJyftQIAEAWbgdYz549tWnTJn399ddKSkpScnKyvvrqK23ZskW9evXKjxoBAMjC7TmwwoUL67vvvtN9993n0r5u3To1a9bshrgWjDkwFHTMgaGgy80cmNtHYCVKlFBQUFCW9qCgIAUHB7s7HAAAeeJ2gL3++uvq37+/jh496mxLSEjQwIEDNXToUI8WBwBATnK1CrFGjRqy2WzO13v37lVYWJhuu+02SdLBgwdlt9t14sQJPfPMM/lTKQAAl8lVgD3++OP5XAYAAO65pguZb1Qs4kBBxyIOFHT5sogDAIAbgdt34sjIyNCkSZO0ZMkSHTx4UOfPn3fZnpiY6LHiAADIidtHYCNGjNDEiRPVrl07JScnq3///mrdurW8vLw0fPjwfCgRAICs3J4Dq1ixoqZOnaoWLVqoaNGiiomJcbb98ssvWrRoUX7VmmvMgaGgYw4MBV2+zIElJCSoWrVqkqQiRYo473/4yCOP6Ouvv3Z3OAAA8sTtALv11ludFzGHh4fr+++/lyRt3rxZdrvds9UBAJADtwOsVatWWrVqlSSpb9++Gjp0qO644w516dJFPXr08HiBAABk55qvA9u0aZM2bNig8PBwPfbYY56q65owB4aCjjkwFHS5mQPz2IXMx44d0/vvv69hw4Z5YrhrQoChoCPAUNBd1wuZExISNGLECE8NBwDAFXEnDgCAJRFgAABLIsAAAJaU63sh9u/f/4rbT5w4cc3FAACQW7kOsOjo6Kv2adCgwTUVAwBAbuU6wFavXp2fdQAA4BbmwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSblahRgbG5vrASMiIvJcDAAAuZWrAIuMjJTNZpNhGLLZbFfsm5GR4ZHCAAC4klydQoyPj9f+/fsVHx+vpUuXqnz58po+fbqio6MVHR2t6dOnq2LFilq6dGl+1wsAgKQ8PA+sdu3aGj58uJo3b+7S/s0332jo0KHaunWrRwvMC54HhoKO54GhoMuX54Ft375d5cuXz9Jevnx57dy5093hAADIE7cDrHLlynrrrbeUmprqbEtLS9Nbb72lypUre7Q4AABykut7IV40c+ZMPfrooypXrpyqV68uSfrtt99ks9n01VdfebxAAACy4/YcmCSdPXtWH3/8sXbt2iXDMFSlShV16NBBhQsXzo8a3cYcGAo65sBQ0OVmDixPAXajI8BQ0BFgKOhyE2Bun0KUpAULFuj999/X/v379fPPPyssLEyTJk1ShQoV1LJly7wM6VGlKzQzuwQgX6WMfMjsEgDTub2IY8aMGerfv78efvhhnTp1ynnhcnBwsCZPnuzp+gAAyJbbATZt2jTNmjVLr732mnx8Lh3A1apVS9u3b/docQAA5MTtAIuPj1eNGjWytNvtdp05c8YjRQEAcDVuB1j58uUVExOTpX3lypWqUqWKJ2oCAOCq3F7EMXDgQD3//PNKTU2VYRj69ddftXjxYo0ZM0azZ8/OjxoBAMjC7QDr3r27HA6HBg0apLNnz6pDhw4qW7aspkyZovbt2+dHjQAAZHFN14GdPHlSmZmZCg0N9WRN1ywk8E6zSwDy1cHXoswuAchXhQbPuWoft+fAGjdurKSkJElSSEiIM7xSUlLUuHFjd4cDACBP3A6wn376SefPn8/SnpqaqnXr1nmkKAAAribXc2CxsbHO33fu3KmEhATn64yMDH377bcqW7asZ6sDACAHuQ6wyMhI2Ww22Wy2bE8VBgQEaNq0aR4tDgCAnOQ6wOLj42UYhipUqKBff/1VJUuWdG7z8/NTaGiovL2986VIAAD+LdcBFhYWJknKzMzMt2IAAMgttxdxjBkzRh999FGW9o8++kjjxo3zSFEAAFyN2wH2/vvv66677srSfvfdd2vmzJkeKQoAgKtxO8ASEhJUunTpLO0lS5bU0aNHPVIUAABX43aAlStXThs2bMjSvmHDBpUpU8YjRQEAcDVu3wuxZ8+eeumll5Senu5cTr9q1SoNGjRIAwYM8HiBAABkx+0AGzRokBITE9W7d2/nHTn8/f01ePBgDRkyxOMFAgCQnTzfzPf06dOKi4tTQECA7rjjDtntdk/XlmfczBcFHTfzRUGXm5v5un0EdlGRIkV077335vXtAABck1wFWOvWrTV37lwFBgaqdevWV+y7bNkyjxQGAMCV5CrAgoKCZLPZnL8DAGC2a3qg5Y2KOTAUdMyBoaDLlwdaAgBwI8jVKcQaNWo4TyFezbZt266pIAAAciNXAfb44487f09NTdX06dNVpUoV1a1bV5L0yy+/aMeOHerdu3e+FAkAwL/lKsDeeOMN5+89e/ZUnz59NHLkyCx9Dh065NnqAADIgdtzYJ999pm6dOmSpb1Tp05aunSpR4oCAOBq3A6wgIAArV+/Pkv7+vXr5e/v75GiAAC4GrfvxPHSSy/pueee09atW1WnTh1JF+bAPvroIw0bNszjBQIAkB23A+yVV15RhQoVNGXKFC1atEiSVLlyZc2dO1ft2rXzeIEAAGSHC5kBC+JCZhR0+XYhc1JSkmbPnq1XX31ViYmJki5c/3X48OG8DAcAgNvcPoUYGxurJk2aKCgoSAcOHFDPnj1VvHhxLV++XH/++afmz5+fH3UCAODC7SOw/v37q1u3btq7d6/LqsOHH35Ya9eu9WhxAADkxO0A27x5s5555pks7WXLllVCQoJHigIA4GrcDjB/f3+lpKRkad+9e7dKlizpkaIAALgatwOsZcuWevPNN5Weni5JstlsOnjwoF555RW1adPG4wUCAJAdtwNswoQJOnHihEJDQ3Xu3Dk1bNhQ4eHhKlq0qEaNGpUfNQIAkIXbqxADAwO1fv16/fjjj9q2bZsyMzNVs2ZNNWnSJD/qAwAgW24FmMPhkL+/v2JiYtS4cWM1btw4v+oCAOCK3DqF6OPjo7CwMGVkZORXPQAA5Irbc2Cvv/66hgwZ4rwDBwAAZnB7Dmzq1Kn6448/VKZMGYWFhalw4cIu27dt2+ax4gAAyInbAdayZUvZbLb8qAUAgFxzO8CGDx+eD2UAAOCeXM+BnT17Vs8//7zKli2r0NBQdejQQSdPnszP2gAAyFGuA+yNN97Q3Llz1aJFC7Vv314//PCDnnvuufysDQCAHOX6FOKyZcv04Ycfqn379pKkTp06KSoqShkZGfL29s63AgEAyE6uj8AOHTqk+vXrO1/Xrl1bPj4+OnLkSL4UBgDAleQ6wDIyMuTn5+fS5uPjI4fD4fGiAAC4mlyfQjQMQ926dZPdbne2paam6tlnn3W5FmzZsmWerRAAgGzkOsC6du2apa1Tp04eLQYAgNzKdYDNmTMnP+sAAMAtbt8LEQCAGwEBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCWZHmDx8fHau3dvlva9e/fqwIED178gAIAlmB5g3bp108aNG7O0b9q0Sd26dbv+BQEALMH0AIuOjlZUVFSW9jp16igmJub6FwQAsATTA8xms+mff/7J0p6cnKyMjAwTKgIAWIHpAVa/fn2NGTPGJawyMjI0ZswY3XfffSZWBgC4kfmYXcD48ePVoEEDVapUSfXr15ckrVu3TikpKfrxxx9Nrg4AcKMy/QisSpUqio2NVbt27XT8+HH9888/6tKli3bt2qWqVauaXR4A4AZl+hGYJJUpU0ajR482uwwAgIWYEmCxsbGqWrWqvLy8FBsbe8W+ERER16kqAICVmBJgkZGRSkhIUGhoqCIjI2Wz2WQYRpZ+NpuNlYgAgGyZEmDx8fEqWbKk83eYq269Wnqhb09Vj7xbpUrfos7/7a2VX/+fc/ugIS+qVZsWKlO2lNLPp+u3mB0aNXKitm25dPTs5+erEaNeUesnHpG/v13r1vysgf2H6+iRY84+27b/qNvCbnX57CkTP9DI4RPyfR8BW5Fi8m3UTt4Vqkk+vjISjylt5Ucyjv15qU+J0vJt2Fbet1WSZFPm30d0fsV0Gf8kSpJ8m3aVd1gV2YoUk9LTlHH4D6X/tERGYoJzDL/WfeR1y22yFQqUUs8o48BOpa/5TMbppOu7wzcBUwIsLCws29//LbujMnheocKF9Pvvu7To42Wat/DdLNv3/RGvwS+/qT8PHJK/v13PPd9dny+fo3sjm+jvv09JkkaNfU1NH26sXt376VTiKb056hUtWvKBHmjQSpmZmc6xxrw1WQvmLnG+PnPmbP7vIGAvJHun15R5ME5pn02UcSZFtuBQKe3S989WrKT8O74qR+xapa5fIaWdk61EaRkZ6c4+mQkHlLHjZxkpf0sBReQb1VL2J19W6syB0v//71XmwV1y/PKVjNPJshUNlu/9T8rv8eeV9vGo673XBZ7pizg6d+6sGTNmqEiRIi7tBw4cUOfOnbVu3TqTKrt5rPphrVb9sDbH7Us/+8rl9euvjlanrm1VpepdWrfmZxUNLKKOXZ5Q76cHae1PF24L9lyvgfotbo0a3l9Pq1etd7739OkzOn78ZP7sCJAD3zrNZaQk6vw3HznbjJS/Xfs0aKOMfbFK/+mzS32ST7j0yfhtzaUXKX8rfd0yBfQYKVtQiIykC30dW753+QzHL1/Lr/WLkpe3lMmUiCeZvox+586dqlatmjZs2OBsmzdvnqpXr65bbrnFxMqQHV9fX3Xt9qSSk1K0Y/suSVJkZFX5+flp9Y+Xgioh4bjidu5V7f/UdHl/n5d6ac+BTVq9/n/q9/Kz8vX1va714+bkHR6pzIR4+bXsrYAXpsi/23B5V29wWQ+bvCtEKPNUguztBijghSmyd35d3nfUyHlQXz/5VLtPmUnHZaQkZt/Hv7C8q9RV5uE/CK98YPoR2KZNm/T666+rcePGGjBggPbu3atvv/1WU6ZMUY8ePa76/rS0NKWlpbm0GUambDbTs7lAeahZI33w0SQVKhSgYwkn9MTj3ZWYeOH0YegtIUpLO6/kpBSX95w4cVKhoSHO1x/MnK/YmB1KSkpRzXsi9PrwAQoLK6eXXnztuu4Lbj62YqHyqdFYjs3fKfXnr+RduoL8Huio8w6HMnZslAoXlc0eIN//tFD6umU6/9MSeZevJr9WLyht8XhlHtrtHMunxv3ybdRONj9/Zf59RGmfTsgSTr4N28qn5gOy+dmVcfgPpX0++Trv8c3B9ADz8fHR2LFjZbfbNXLkSPn4+GjNmjWqW7durt4/ZswYjRgxwqUtwK+4CtlL5Ee5N631azfp/vtaqniJYHXu2k6z505W08ZtdfJkDn95SrLJdXXpzPfmOn/fuWO3kpKSNffjdzXijbd1KjEpH6vHTc9mU2bCAaWvXSpJchw/KFtIGfnUuF8ZOzY6/+DN+CPaeQrQcfyQvMqGyyeykc5fFmCOHb8o48BO2QoHyad2M9lb9lbqx6OkDIezT/qvK+WIXStbUIkL82SP9CLE8oHphynp6ekaMGCAxo0bpyFDhqhu3bpq1aqVvvnmm1y9f8iQIUpOTnb5CfALzueqbz5nz55T/P6D2rr5N730wmvKyMhQxy5tJUnHj52U3e6noGKBLu8JKVlCJ078nd1wkqQtm3+TJJWvcFv+FQ5IMk4nyTh5xLXt76OyBV74Q9c4+4+MDIcyr9DH6fw5GaeOKfOvPTq/4j3ZipeW9533uPY5d/pCnwM7df6LmfKuWF1eZSp6fL9udqYHWK1atfTFF1/op59+0qhRo/TTTz+pX79+at26tXr37n3V99vtdgUGBrr8cPrwOrDZZLf7SZJiYn7X+fPn1ej+S4/FueWWkqpc5Q79umlbjkNEVK8sSTqWcCLHPoAnZB7+Q7bipVzabMVvubSQIzNDmQkH5HWlPjmxSfK+0sks24X/uWIf5IXp/0Rr1aqlqVOnqnDhwpIuXLw8ePBgNW3aVJ06dTK5uptD4cKFVL7CZZc23H6rqlarrFOnknQqMUn9Xn5O365cpWMJJ1S8eDH16NlRZcqU0v+Wr5Qk/ZNyWgvnf643R72ixMQkJZ1K0oi3XtHOHXu0ZvWFVYm1akeq1r2RWr92k1JS/lGNmtX01phXtfLr/9Phv46ast+4eTg2fy97p1flU6eFMnZtllfpCvKp3kjnv5t7qc+mlfJr+Zy8/9qtzD93ybtCNXmHRypt0ThJki2opLwr11ZG/O/S2X9kKxosn/80lxzpyth/4ZpIr9Ll5VW6gjL+2iOlnpWtWEn53tdKmaeOKfPIPjN2vUCzGTfwxVZpaWmy2+1uvy8k8M58qKbgirqvtv73zcdZ2hcvXKaXXxqm9z+cqHtqVVfxEsE6lXhK0du2a+LbMxS9bbuzr93up+FvDVabto/I39/feSHzkcMXLvCMqF5F4ycO1x13VJCf3U9/HTqi5Uu/1rTJs3TuXOr12tUC4+BrWR8Ciyvzqlhdfg2fkC34FhnJJ5S++Ttl/OZ6+Yh3tfryrdNCtqLBMhITlL5+hTL+iJZ04UJov2bd5VUqTPIvLONMijIP7Vb6xi+cFzLbQm6VX5MO8gotJ/naZZxOUkb8djk2fsmFzG4qNHjOVfvcEAG2Zs0aTZgwQXFxcbLZbKpcubIGDhzofLyKuwgwFHQEGAq63ASY6ZNFH3/8sZo0aaJChQqpT58+euGFFxQQEKAHHnhAixYtMrs8AMANyvQjsMqVK+vpp59Wv379XNonTpyoWbNmKS4uzu0xOQJDQccRGAo6SxyB7d+/X48++miW9scee4wb/QIAcmR6gJUrV06rVq3K0r5q1SqVK1fOhIoAAFZg2jL6Hj16aMqUKRowYID69OmjmJgY1atXTzabTevXr9fcuXM1ZcoUs8oDANzgTJsD8/b21tGjRxUaGqrly5frnXfecc53XVyF2LJlyzyNzRwYCjrmwFDQ5WYOzLQjsMtzs1WrVmrVqpVZpQAALMjUOTCbzWbmxwMALMzUW0ndeeedVw2xxMSc73YOALh5mRpgI0aMUFBQkJklAAAsytQAa9++vUJDQ80sAQBgUabNgTH/BQC4FqYF2A1wD2EAgIWZdgoxMzPTrI8GABQApt9KCgCAvCDAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAAS7IZhmGYXQSsLS0tTWPGjNGQIUNkt9vNLgfwKL7fNy4CDNcsJSVFQUFBSk5OVmBgoNnlAB7F9/vGxSlEAIAlEWAAAEsiwAAAlkSA4ZrZ7Xa98cYbTHCjQOL7feNiEQcAwJI4AgMAWBIBBgCwJAIMAGBJBBhuSHPnzlWxYsXMLgO4ogMHDshmsykmJsbsUm5KBFgB0q1bN9lsNo0dO9alfcWKFbLZbG6Ndfvtt2vy5Mk5bv/pp59ks9mu+DN37tw87MUFTz75pPbs2ZPn9wMXXe172q1btzyPXa5cOR09elRVq1b1XMHINR+zC4Bn+fv7a9y4cXrmmWcUHBycb59Tr149HT161Pm6b9++SklJ0Zw5c5xtQUFBeR4/ICBAAQEB11QjIMnle/rpp59q2LBh2r17t7PtWr5n3t7eKlWq1DXVh7zjCKyAadKkiUqVKqUxY8Zcsd/SpUt19913y2636/bbb9c777zj3NaoUSP9+eef6tevn/Ov1H/z8/NTqVKlnD8BAQGy2+3O18HBwRo8eLBCQ0Pl7++v++67T5s3b5Ykpaam6u6779bTTz/tHC8+Pl5BQUGaNWuWpOxPIX7xxReqVauW/P39FRISotatW+f1HxNuIpd/T4OCgmSz2VzaFi1apIoVK8rPz0+VKlXSggULnO/t0aOHIiIilJaWJklKT0/XPffco44dO0rK/hTijh071KJFCwUGBqpo0aKqX7++9u3bd133+WZBgBUw3t7eGj16tKZNm6a//vor2z5bt25Vu3bt1L59e23fvl3Dhw/X0KFDnaf8li1bpltvvVVvvvmmjh496vIXbG4NGjRIS5cu1bx587Rt2zaFh4eradOmSkxMlL+/vxYuXKh58+ZpxYoVysjIUOfOnXX//ferV69e2Y739ddfq3Xr1mrRooWio6O1atUq1apVy+26gMstX75cffv21YABA/T777/rmWeeUffu3bV69WpJ0tSpU3XmzBm98sorkqShQ4fq5MmTmj59erbjHT58WA0aNJC/v79+/PFHbd26VT169JDD4bhu+3RTMVBgdO3a1WjZsqVhGIZRp04do0ePHoZhGMby5cuNy/+v7tChg/Hggw+6vHfgwIFGlSpVnK/DwsKMSZMm5emzT58+bfj6+hoLFy50bj9//rxRpkwZY/z48c628ePHGyEhIcaLL75olCpVyjhx4oRz25w5c4ygoCDn67p16xodO3bMdT1Adv79vapXr57Rq1cvlz5t27Y1mjdv7ny9ceNGw9fX1xg6dKjh4+NjrFmzxrktPj7ekGRER0cbhmEYQ4YMMcqXL2+cP38+X/cDF3AEVkCNGzdO8+bN086dO7Nsi4uLU1RUlEtbVFSU9u7dq4yMjGv+7H379ik9Pd3lM3x9fVW7dm3FxcU52wYMGKBKlSpp2rRpmjNnjkJCQnIcMyYmRg888MA11wZcLqd/Fy7/ntatW1cvv/yyRo4cqQEDBqhBgwY5jhcTE6P69evL19c332rGJQRYAdWgQQM1bdpUr776apZthmFkmdcyPHhHsYtjZfcZl7cdP35cu3fvlre3t/bu3XvFMVnQgfxyte9pZmamNmzYwPf0BkSAFWBjx47Vl19+qY0bN7q0V6lSRevXr3dp27hxo+688055e3tLurBII69HY+Hh4fLz83P5jPT0dG3ZskWVK1d2tvXo0UNVq1bV/PnzNWjQoGyPFi+KiIjQqlWr8lQPkJPKlStn++/C5d/Tt99+W3FxcVqzZo2+++47l5W2/xYREaF169YpPT0932rGZUw9gQmPunwe6qLOnTsb/v7+LnNgW7duNby8vIw333zT2L17tzF37lwjICDAmDNnjrPPgw8+aDz22GPGX3/95TI3ldvP7tu3r1GmTBlj5cqVxo4dO4yuXbsawcHBRmJiomEYhvHuu+8axYoVMw4ePGgYhmF06tTJiIyMNNLS0gzDyDpXsXr1asPLy8sYNmyYsXPnTiM2NtYYN26cm/+EcLP79/dq+fLlhq+vrzFjxgxjz549xjvvvGN4e3sbq1evNgzDMKKjow0/Pz/jiy++MAzDMGbPnm0ULVrU2Ldvn2EYWefATp48aZQoUcJo3bq1sXnzZmPPnj3G/PnzjV27dl3P3bxpEGAFSHYBduDAAcNutxv//lvl888/N6pUqWL4+voat912m/H222+7bP/555+NiIiIbN+bm88+d+6c8eKLLxohISGG3W43oqKijF9//dUwDMOIi4szAgICjEWLFjn7JycnG7fffrsxaNAgwzCy/ofGMAxj6dKlRmRkpOHn52eEhIQYrVu3vmpdwOWy+15Nnz7dqFChguHr62vceeedxvz58w3DuPAdrlKlivH000+79G/VqpVRr149w+FwZAkwwzCM3377zXjooYeMQoUKGUWLFjXq16/vDDx4Fo9TAQBYEnNgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWDATSa7h4Xmhc1m04oVK655HCCvCDAgFy4+mTqnn27dul23Wrp166bHH3/8un0ecKPyMbsAwAoufyr1p59+qmHDhmn37t3Otn8/RiM9PZ1nQgH5jCMwIBdKlSrl/AkKCpLNZnO+Tk1NVbFixbRkyRI1atRI/v7++vjjjzV8+HBFRka6jDN58mTdfvvtLm1z5sxR5cqV5e/vr7vuuivHx9Xn1sSJE1WtWjUVLlxY5cqVU+/evXX69Oks/VasWKE777xT/v7+evDBB3Xo0CGX7V9++aXuuece+fv7q0KFChoxYoQcDsc11QZ4EgEGeMjgwYPVp08fxcXFqWnTprl6z6xZs/Taa69p1KhRiouL0+jRozV06FDNmzcvz3V4eXlp6tSp+v333zVv3jz9+OOPGjRokEufs2fPatSoUZo3b542bNiglJQUtW/f3rn9u+++U6dOndSnTx/t3LlT77//vubOnatRo0bluS7A48y+HT5gNf9+JMfFR2pMnjzZpd8bb7xhVK9e3aVt0qRJRlhYmPN1uXLlXB4rYxiGMXLkSKNu3bo5fn52j825kiVLlhglSpRwqV+S8csvvzjb4uLiDEnGpk2bDMMwjPr16xujR492GWfBggVG6dKlna8lGcuXL891HYCnMQcGeEitWrXc6n/ixAkdOnRITz31lHr16uVsdzgcCgoKynMdq1ev1ujRo7Vz506lpKTI4XAoNTVVZ86cUeHChSVJPj4+LvXeddddKlasmOLi4lS7dm1t3bpVmzdvdjniysjIUGpqqs6ePatChQrluT7AUwgwwEMuhsNFXl5eMv71uL3LHzWfmZkp6cJpxP/85z8u/by9vfNUw59//qnmzZvr2Wef1ciRI1W8eHGtX79eTz31VJbH3Ntstizvv9iWmZmpESNGqHXr1ln6+Pv756k2wNMIMCCflCxZUgkJCTIMwxkMMTExzu233HKLypYtq/3796tjx44e+cwtW7bI4XDonXfekZfXhSnuJUuWZOnncDi0ZcsW1a5dW5K0e/duJSUl6a677pIk1axZU7t371Z4eLhH6gLyAwEG5JNGjRrpxIkTGj9+vJ544gl9++23WrlypQIDA519hg8frj59+igwMFAPP/yw0tLStGXLFp06dUr9+/fPcezk5GSXMJSk4sWLq2LFinI4HJo2bZoeffRRbdiwQTNnzszyfl9fX7344ouaOnWqfH199cILL6hOnTrOQBs2bJgeeeQRlStXTm3btpWXl5diY2O1fft2vfXWW575BwRcK7Mn4QCryWkRR3R0dJa+M2bMMMqVK2cULlzY6NKlizFq1CiXRRyGYRgLFy40IiMjDT8/PyM4ONho0KCBsWzZshw/v2vXroakLD9du3Y1DMMwJk6caJQuXdoICAgwmjZtasyfP9+QZJw6dcql/qVLlxoVKlQw/Pz8jMaNGxsHDhxw+Zxvv/3WqFevnhEQEGAEBgYatWvXNj744APndrGIAyazGca/TtIDAGABXAcGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCS/h/okACgaCyxDAAAAABJRU5ErkJggg==",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "mat = confusion_matrix(y_test, y_pred_class)\n",
        "sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False, \n",
        "            xticklabels=['Not Toxic', 'Toxic'], yticklabels=['Not Toxic', 'Toxic'])\n",
        "plt.xlabel('True Label')\n",
        "plt.ylabel('Predicted Label')\n",
        "\n",
        "metrics.confusion_matrix(y_test, y_pred_class)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "w-yTunCvYFS9",
        "outputId": "b72a063e-4a5c-4dc6-be98-cc705efbf0f0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.90      0.87      0.88      9800\n",
            "           1       0.84      0.88      0.86      7570\n",
            "\n",
            "    accuracy                           0.87     17370\n",
            "   macro avg       0.87      0.87      0.87     17370\n",
            "weighted avg       0.87      0.87      0.87     17370\n",
            "\n"
          ]
        }
      ],
      "source": [
        "print(classification_report(y_test, y_pred_class))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {
        "id": "fsjvOBwVYFS9",
        "outputId": "8318f853-3cb9-409f-e740-e1aa465e700d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[7.55255859e-03, 9.92447441e-01],\n",
              "       [1.15466699e-06, 9.99998845e-01],\n",
              "       [4.16633513e-10, 1.00000000e+00],\n",
              "       ...,\n",
              "       [8.71676839e-02, 9.12832316e-01],\n",
              "       [1.00000000e+00, 2.77420961e-11],\n",
              "       [9.93314764e-01, 6.68523605e-03]])"
            ]
          },
          "execution_count": 63,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "raw = mnb.predict_proba(X_test_dtm)\n",
        "raw"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {
        "id": "0YqaXFCnYFS9",
        "outputId": "ea478193-3118-4648-a230-087f7b0da4fb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([7.55255859e-03, 1.15466699e-06, 4.16633513e-10, ...,\n",
              "       8.71676839e-02, 1.00000000e+00, 9.93314764e-01])"
            ]
          },
          "execution_count": 64,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the predicted probabilities for the X_test_dtm (non-toxic class).\n",
        "\n",
        "y_pred_prob_non_toxic_class = mnb.predict_proba(X_test_dtm)[:, 0]\n",
        "y_pred_prob_non_toxic_class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {
        "id": "Nc8vKTgRYFS9",
        "outputId": "88bf7d90-1abb-44ae-d08f-f3da09a13d84"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([9.92447441e-01, 9.99998845e-01, 1.00000000e+00, ...,\n",
              "       9.12832316e-01, 2.77420961e-11, 6.68523605e-03])"
            ]
          },
          "execution_count": 65,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the predicted probabilities for the X_test_dtm (toxic class).\n",
        "\n",
        "y_pred_prob_toxic_class = mnb.predict_proba(X_test_dtm)[:, 1]\n",
        "y_pred_prob_toxic_class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 66,
      "metadata": {
        "id": "p2wFoXqrYFS9",
        "outputId": "a286de4a-c1a6-4b74-e3c8-d948d6f98a25"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Di ako makapaniwala na pangalawa si Mar Roxas ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>7.552559e-03</td>\n",
              "      <td>9.924474e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A lot of you niggas turned these bitches into ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.154667e-06</td>\n",
              "      <td>9.999988e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>RT @ZitlalyL: @Vic_Manzan0 just like you&amp;#1280...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4.166335e-10</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>our playground for the morning.  big bear tri ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.396200e-01</td>\n",
              "      <td>6.038000e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"@ellisluciano: Binay as President #ShortHorro...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>7.096837e-01</td>\n",
              "      <td>2.903163e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17365</th>\n",
              "      <td>Mad respect to Affleck for refusing to wear a ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.530769e-01</td>\n",
              "      <td>4.692311e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17366</th>\n",
              "      <td>RT @TAXSTONE: If YA bitch ex is a nigga u thin...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5.077755e-13</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17367</th>\n",
              "      <td>@imcessfederico trillion yan ! Wala eh... Only...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>8.716768e-02</td>\n",
              "      <td>9.128323e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17368</th>\n",
              "      <td>fathersdaymessage  #fathers #day #message buy...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.000000e+00</td>\n",
              "      <td>2.774210e-11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17369</th>\n",
              "      <td>@user my woodpecker shot in this months issue...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.933148e-01</td>\n",
              "      <td>6.685236e-03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>17370 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "0      Di ako makapaniwala na pangalawa si Mar Roxas ...       1          1   \n",
              "1      A lot of you niggas turned these bitches into ...       1          1   \n",
              "2      RT @ZitlalyL: @Vic_Manzan0 just like you&#1280...       1          1   \n",
              "3      our playground for the morning.  big bear tri ...       0          0   \n",
              "4      \"@ellisluciano: Binay as President #ShortHorro...       1          0   \n",
              "...                                                  ...     ...        ...   \n",
              "17365  Mad respect to Affleck for refusing to wear a ...       0          0   \n",
              "17366  RT @TAXSTONE: If YA bitch ex is a nigga u thin...       1          1   \n",
              "17367  @imcessfederico trillion yan ! Wala eh... Only...       0          1   \n",
              "17368   fathersdaymessage  #fathers #day #message buy...       0          0   \n",
              "17369   @user my woodpecker shot in this months issue...       0          0   \n",
              "\n",
              "       prob_non_toxic    prob_toxic  \n",
              "0        7.552559e-03  9.924474e-01  \n",
              "1        1.154667e-06  9.999988e-01  \n",
              "2        4.166335e-10  1.000000e+00  \n",
              "3        9.396200e-01  6.038000e-02  \n",
              "4        7.096837e-01  2.903163e-01  \n",
              "...               ...           ...  \n",
              "17365    9.530769e-01  4.692311e-02  \n",
              "17366    5.077755e-13  1.000000e+00  \n",
              "17367    8.716768e-02  9.128323e-01  \n",
              "17368    1.000000e+00  2.774210e-11  \n",
              "17369    9.933148e-01  6.685236e-03  \n",
              "\n",
              "[17370 rows x 5 columns]"
            ]
          },
          "execution_count": 66,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Build a data frame with the input text, actual class, predicted class, and the predicted probabilities.\n",
        "\n",
        "test_results_df = pd.DataFrame({'text': X_test, 'actual': y_test, 'predicted': y_pred_class, 'prob_non_toxic': y_pred_prob_non_toxic_class, 'prob_toxic': y_pred_prob_toxic_class})\n",
        "\n",
        "# Reset the index.\n",
        "test_results_df.reset_index(drop=True, inplace=True)\n",
        "\n",
        "test_results_df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "u-bSN6iOYFS-"
      },
      "source": [
        "# 5 | Using Other AI Model (for comparison)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 67,
      "metadata": {
        "id": "XjXjHN6jYFS-",
        "outputId": "9e1d3b7e-201e-48a8-a3a1-046a46f82816"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Inaasahan na ni Vice President Jejomar Binay n...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Salamat sa walang sawang suporta ng mga taga m...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>@rapplerdotcom putangina mo binay TAKBO PA</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Binay with selective amnesia, forgetting about...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69475</th>\n",
              "      <td>when baylor wins im talking so much trash</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69476</th>\n",
              "      <td>who's downie like a brownie 4 brunch? Need mim...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69477</th>\n",
              "      <td>yea so about @N_tel 's new friend.. all my fri...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69478</th>\n",
              "      <td>you know what they say, the early bird gets th...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>69479</th>\n",
              "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>69480 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  label\n",
              "0      Inaasahan na ni Vice President Jejomar Binay n...      0\n",
              "1      Mar Roxas TANG INA TUWID NA DAAN DAW .. EH SYA...      1\n",
              "2      Salamat sa walang sawang suporta ng mga taga m...      0\n",
              "3             @rapplerdotcom putangina mo binay TAKBO PA      1\n",
              "4      Binay with selective amnesia, forgetting about...      0\n",
              "...                                                  ...    ...\n",
              "69475          when baylor wins im talking so much trash      0\n",
              "69476  who's downie like a brownie 4 brunch? Need mim...      0\n",
              "69477  yea so about @N_tel 's new friend.. all my fri...      0\n",
              "69478  you know what they say, the early bird gets th...      0\n",
              "69479  you've gone and broke the wrong heart baby, an...      0\n",
              "\n",
              "[69480 rows x 2 columns]"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "loVbPgD8YFS-"
      },
      "source": [
        "## 5.1 | K-Means using 2 clusters."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "1BjJMirPdQUV"
      },
      "source": [
        "The K-means models will use the X_train_dtm as its training set. The K-means algorithm aims to label unlabeled data, which in this case, the labeled training set will be considered unlabeled data. This data will then be compared to the actual labeled values in the y_train set, hence the increase in the number of test values when compared to the test values of the MNB models.\n",
        "\n",
        "Increasing the unlabeled data to contain the entire dataset results to a negligible difference in accuracy rate but a significant increase in runtime, hence only the test dataset, which contains 80% of the dataset, is used."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {
        "id": "GIPgyQltYFS-",
        "outputId": "13ede6da-c07f-4580-e09a-84ec03208ab5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2, 68698)"
            ]
          },
          "execution_count": 68,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.cluster import KMeans\n",
        "\n",
        "kmeans = KMeans(n_clusters=2, n_init=10, random_state=42)\n",
        "clusters = kmeans.fit_predict(X_train_dtm)\n",
        "kmeans.cluster_centers_.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {
        "id": "Nuur0wo-YFS-"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from scipy.stats import mode\n",
        "\n",
        "labels = np.zeros_like(clusters)\n",
        "for i in range(2):\n",
        "    mask = (clusters == i)\n",
        "    labels[mask] = mode(y_train[mask], keepdims=True)[0]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "TKGiI2GPYFS_"
      },
      "source": [
        "### Check for accuracy."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "metadata": {
        "id": "n6_d7WlAYFS_",
        "outputId": "d6596641-ed5d-4cf3-e02c-44bdd4cf89f8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.6751871042026483"
            ]
          },
          "execution_count": 70,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "accuracy_score(y_train, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 71,
      "metadata": {
        "id": "syOh1cEsYFS_",
        "outputId": "073525c8-44c2-4227-94db-4c07356cf513"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(113.9222222222222, 0.5, 'Predicted label')"
            ]
          },
          "execution_count": 71,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAGwCAYAAADITjAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAy7UlEQVR4nO3de3zP9f//8ft756MN21o0Z2EYlU9hOSQl9cmpQs7NKSWSkMopFUZhSooykj4qh0/6lH59yPkbOQwxh5gzscYw7Pj6/bG8P9aGvec9r73mdr1cdvm838/X8/V6Pd77vHPf6/V8vl4vm2EYhgAAsBgXswsAAKAgCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJbmYXUBjSEw+YXQJQqE627G12CUChCvt1+Q37cAQGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACzJ9AB7+umnNX78+FztEydO1DPPPGNCRQAAKzA9wFatWqUnnngiV/tjjz2m1atXm1ARAMAKTA+wCxcuyMPDI1e7u7u7zp07Z0JFAAArMD3AatWqpQULFuRq/9e//qXw8HATKgIAWIGb2QWMGDFCTz31lPbv369mzZpJkpYvX64vv/xSX3/9tcnVAQCKKtMDrFWrVlqyZIneffddffPNN/L29lZERIT++9//qkmTJmaXBwAoomyGYRhmF+Fs6YkHzC4BKFQnW/Y2uwSgUIX9uvyGfUwfAwMAoCBMOYVYqlQp7d27V0FBQSpZsqRsNts1+yYlJd3CygAAVmFKgE2ePFn+/v7219cLMAAA8sIYGGBBjIGhuLPEGNinn36aZ3tGRoaGDx9+i6sBAFiF6QE2ePBgPfXUUznGunbv3q37779fX331lYmVAQCKMtMDbOvWrfrjjz9Uu3Zt/fTTT/rwww917733qlatWoqLizO7PABAEWX6hcwVK1bU6tWrNWjQID322GNydXXV3Llz1bFjR7NLAwAUYaYfgUnSd999py+//FINGzZUYGCgZs6cqePHj5tdFgCgCDM9wPr27av27dtr6NChWr16tbZv3y5PT0/Vrl2bMTAAwDWZfgpx3bp12rBhg+rUqSNJCg0N1ffff68PP/xQUVFRat++vckVAgCKItOvA0tNTZWnp2eey/bs2aNq1ao5vE2uA0Nxx3VgKO7ycx2Y6Udgnp6eOnv2rD799FPFx8fLZrOpRo0a6tmzZ4HCCwBwezB9DGzTpk2qXLmyJk+erKSkJCUmJmry5MmqXLmytmzZYnZ5AIAiyvRTiI0aNVKVKlU0c+ZMubllHxBmZGSoV69eOnDggFavXu3wNjmFiOKOU4go7ixxCnHTpk05wkuS3NzcNHToUNWrV8/EygAARZnpAVaiRAkdPnxY1atXz9F+5MgR+x3r4Twz5y7Qf1etU8Kho/Ly9FDd2uEa1C9KFcvfZe+TmHRGk6d/pvUbt+j8hRTdV7eWXh/UT+XDykqSks+d14ezPtf6jVt08lSiAgNLqFmjBnqpdzf5+/nat/PoU911/OSpHPvv2eUZDeoXJUk6m3xOw8ZEa+/vCTp77pxKlQxUswcbaODz3eXn6yugoDzvqS3/rh3kUb2qXIODlPjqSF1atS5HH7cK5RT4Um953hsh2VyUfuCg/hw+Vpl/ZH9nXUqXVOCAvvJ64D7ZfLyVceiozs2er0sr/ndWyL1aVQW+1Fse4dVkZGbp0s+rdXbyRzIuXbb3CRz8ojzr1JJ75QpKP3hYf3Tue2t+CbcB0wJs7ty56tChgzp06KCePXtq0qRJatiwoWw2m9auXashQ4bo2WefNau8YmtT3A492+5J1apxtzIyMxXzyRz1GfSG/v3Fx/Lx9pJhGBr42ltyc3NTzISR8vPx1dwFi9Rr4Ov2PqcS/9SpxCS92r+XKlUopxN/nNJbEz/Q6cQ/NfmdN3Psr3+vrnq61WP29z7e3vbXNptNDzWqr5d6d1OpkgE6fPS43nlvupInnlf06GG37HeC4sfm7a20vfuVsnSZgqLH5FruWvZOhcycqpRvf1Dyx3OUlZIi9wrlZKSl2fuUHjNcNj9fJb7ypjKTz8m3RTOVfvdN/dHtBaXv/V0uQaUV/GG0Lv20UmcmxsjF11eBr7ygUqOG6c/Xcu4zZekyedSsLveqlQr9s99OTAuw5557To899pgmTZokm82mbt26KSMjQ5Lk7u6ufv36afz48WaVV2x9/P7bOd6//fogNf7ns9q1Z5/q1a2tQ0eOadvO3Vry+QxVqVRekvTm4BfV+J/P6vufVurpVo+paqUKmvLu/4Kq3F1lNKBPd732VrQyMjLl5uZqX+br462g0qXyrCWghL86tv2n/X2Z0DvUod0/NXv+N878yLgNXV6/UZfXb7zm8sAXeury+g1KnvaJvS3z2IkcfTxqh+vM+ClK27VHknTusy/k9+zT8qheVel7f5d3o/pSRqbORMdIf00lOBMdo9AvPlHyXWWUcTT7bkJn3/tQklQiMIAAczLTZiFemTvi4eGhqVOn6syZM4qLi9PWrVuVlJSkyZMnX/P6MDjPhZSLkrLDRJLS0tMlSR4e7vY+rq6ucnd309btO6+5nfMXUuTn65MjvCTp0y++VmTL9nqq+4v6eM6XSv9r+3k5dfpP/XfVOtWrW7vAnwe4IZtNXpEPKOPwUQXFjFeZH79RyOwP5N0kMke31G075PPIQ3Ip4S/ZbPJ+5CHZPNx1eXNc9mbc3WVkpNvDS5KM1FRJkgff4VvC1Gn0Vz+J2cfHR7Vr11ZERIR8fHxMrOr2YRiGomM+0b0RNVW1UgVJUsXyYSoTGqKpH8cq+dx5paena9bnXynxzzM6/WdSnts5m3xOH8d+qWdaP56jvcszbTRxzGv6bNoEPfvUk5r31RKNnfRhrvWHjBqves3aqFmbLvLz8dFbr73s7I8K2LmUCpSLr4/8u3fU5f/7VadfGqZLK9eqdPTo7PGwv/w5/G3JzUVlly/RXeuXqdTrLytxyCj7kdrlTVvlWrqU/Lu0l9zcZPP3U8ALPSVJrkF5n3WAc5k6iaNHjx43PMpatGjRdZenpqYq9a+/eq5wuc7dPfA/77w/XXv3J2juR5Psbe5ubpr8zpsaOW6KIlu2l6uri+rXu0eN6uc9I/RCSopeeHWkKlcsp35RnXMs69axrf11tSoVFeDvp0FvvqNXXohSYEAJ+7JhA/qoX1RnHTx8VFNnxCp62ica8Wp/J39a4C+27L/bL61arwtfLpQkpe/dL8+ImvJt96RSt2yXJAX0e04u/v469cKryjqbLO8mkQoaP1Kner+s9P0JyjhwSEmjJyhwUD8FvNhLysrU+QWLlflnkpSZZdrHu52YGmD+/v7yvmpQvyDGjRunMWNyDpi+OWSARg4deFPbLe7efX+6fl77i+Z8OFGhIcE5ltWsXlUL53yo8xdSlJ6erlIlA/Vs75dVs3rVHP1SUi6q7ysj5OPjranvjpC72/W/ThG1smeaHj56PEeABZUupaDSpVSpfJgCS/ir2wtD9HyPTgrmr1gUgqyzyTIyMpSRcChHe3rCYXnWrSUpe5KHf4e2OtEhShkHsvul7zsgz3tqy++Z1jozfook6eKPK3TxxxVyKVVSxqVLkiH5d3paGcdzjqehcJgaYDExMQoJCbmpbQwfPlyvvPJKjjaX88duapvFmWEYevf9j7R89XrN/mCC7ioTes2+V6bEHzpyTDt371P/Xl3tyy6kpKjvoDfl7uGuaRNGydPT44b7jt+7X5IUfI1JHdn1Zf9v2nXGyoCbkpGhtF175FY+LEezW7m7lHHiD0mSi5dXdmNWzvs8GJlZkotNf5eVdEaS5PvkYzLS0nR5w+ZCKBx/Z1qAXT3+dTM8PT1znS5MT0t0yraLo7ff+1Df/7RSMeNHytfHW4l/jWv5+fnK66/f448r1qhkYIDuvCNY+w4c1PgpM9SsUQNFPnCfpOwjrz4vv6FLqamaOnKIUlIuKuWvySAlAwPk6uqquN/itf233br/3gj5+fnqt/i9io75RA89WF93hmb/0bJ6/Ub9eeasatW4Wz7e3tqfcEjvT/9M90SEq+ydd5jw20FxYfP2kttf1y1KkmuZULnfXVlZyeeV+ccpnf98gUq/O0KpW7crdVOcvBr8Q96NGujU89l/DKcfPKz0w0dVcvggJU+doczkc/Jp+qC8HrhPiYPesG/X75nWSt2+S8alS/J64D4FDOij5A9mybiQYu/jdlcZ2Xy85Vq6lGyennK/u3L2Pg4ckv6aeY2CMe1WUi4uLjp58uRNH4HlhVtJXVutyJZ5tr/9+itq88QjkqR5X/9bs+d/oz+Tziq4dCm1euxhPf/cs3J3z56ZuHHLdkW9lPd1Wj9+E6uyd96hXXt+19uTPlDC4aNKS0tXmdAQPda8iaI6Py3vv/663bh5m6Z+MkcHDh5WWlq6Qu8IVvMmDdWzS3uV8PcrhE9ffHArqevzvLeOQj5+P1d7ync/KmlMtKTsoyX/Hs/KNSRYGYePKPnjObq8er29r1tYWQX07yXPOrVl8/FSxpHjOj/vK1384b/2PqVGD5NXZH25+Hgp/eCRXMslKXjGe/K6r26uWo636qTMv474kFt+biVlWoCtWrVKkZGROW4h5SwEGIo7AgzFXZG+F2KTJk3M2jUAoBgw/XEqAAAUBAEGALAkAgwAYEmmB5irq6tOnTqVq/3PP/+Uq6trHmsAAFAEAuxakyBTU1Pl4XHji2MBALcn02YhxsTESMq+oHnWrFny8/vfdT+ZmZlavXp1rodcAgBwhWkBNnnyZEnZR2AzZszIcbrQw8NDFSpU0IwZM8wqDwBQxJkWYAkJCZKkhx56SIsWLVLJkiXNKgUAYEGm3sxXkn7++Wf76yvjYc66TyIAoPgyfRKHJM2dO1e1a9eWt7e3vL29FRERoc8//9zssgAARZjpR2Dvv/++RowYof79+ysyMlKGYWjdunV6/vnnlZiYqEGDBpldIgCgCDLtZr5XVKxYUWPGjFG3bt1ytM+ZM0ejR4+2j5U5gpv5orjjZr4o7vJzM1/TTyGeOHFCDRs2zNXesGFDnTjBU00BAHkzPcCqVKmir776Klf7ggULVLVq1TzWAACgCIyBjRkzRh06dNDq1asVGRkpm82mtWvXavny5XkGGwAAUhE4Anvqqae0YcMGBQUFacmSJVq0aJGCgoK0ceNGtW3b1uzyAABFlOmTOAoDkzhQ3DGJA8WdJSZxAABQEKaNgbm4uNzwjhs2m00ZGRm3qCIAgJWYFmCLFy++5rL169dr2rRp13zUCgAApgVY69atc7Xt3r1bw4cP19KlS9W5c2eNHTvWhMoAAFZQJMbAjh8/rt69eysiIkIZGRmKi4vTnDlzVK5cObNLAwAUUaYGWHJysoYNG6YqVapo586dWr58uZYuXapatWqZWRYAwAJMO4UYHR2tCRMmKDQ0VF9++WWepxQBALgW064Dc3Fxkbe3t5o3b57jacx/t2jRIoe3zXVgKO64DgzFXX6uAzPtCKxbt248uBIAUGCmBVhsbKxZuwYAFANFYhYiAACOIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlpSvW0l9++23+d5gq1atClwMAAD5la8Aa9OmTb42ZrPZlJmZeTP1AACQL/kKsKysrMKuAwAAh9zUGNjly5edVQcAAA5xOMAyMzM1duxYlS1bVn5+fjpwIPvhkSNGjNCnn37q9AIBAMiLwwH2zjvvKDY2VtHR0fLw8LC3165dW7NmzXJqcQAAXIvDATZ37lx98skn6ty5s1xdXe3tERER2r17t1OLAwDgWhwOsGPHjqlKlSq52rOyspSenu6UogAAuBGHA6xmzZpas2ZNrvavv/5a99xzj1OKAgDgRvI1jf5qo0aNUteuXXXs2DFlZWVp0aJF2rNnj+bOnavvvvuuMGoEACAXh4/AnnzySS1YsEDff/+9bDabRo4cqfj4eC1dulSPPPJIYdQIAEAuNsMwDLOLcLb0xANmlwAUqpMte5tdAlCown5dfsM+Dp9CvGLTpk2Kj4+XzWZTjRo1dN999xV0UwAAOMzhADt69KieffZZrVu3ToGBgZKks2fPqmHDhvryyy8VFhbm7BoBAMjF4TGwqKgopaenKz4+XklJSUpKSlJ8fLwMw1DPnj0Lo0YAAHJx+AhszZo1Wr9+vapVq2Zvq1atmqZNm6bIyEinFgcAwLU4fARWrly5PC9YzsjIUNmyZZ1SFAAAN+JwgEVHR+ull17Spk2bdGUC46ZNmzRw4EBNmjTJ6QUCAJCXfE2jL1mypGw2m/19SkqKMjIy5OaWfQbyymtfX18lJSUVXrX5xDR6FHdMo0dx57Rp9FOmTLnZWgAAcKp8BVj37t0Luw4AABxS4AuZJenSpUu5JnSUKFHipgoCACA/HJ7EkZKSov79+yskJER+fn4qWbJkjh8AAG4FhwNs6NChWrFihaZPny5PT0/NmjVLY8aMUZkyZTR37tzCqBEAgFwcPoW4dOlSzZ07V02bNlVUVJQaNWqkKlWqqHz58vriiy/UuXPnwqgTAIAcHD4CS0pKUsWKFSVlj3ddmTb/4IMPavXq1c6tDgCAa3A4wCpVqqSDBw9KksLDw/XVV19Jyj4yu3JzXwAACpvDAfbcc89p27ZtkqThw4fbx8IGDRqkIUOGOL1AAADyctMPtDx8+LA2bdqkypUrq06dOs6q66ZwJw4Ud9yJA8VdoT7Q8opy5cqpXLlyN7sZAAAckq8Ai4mJyfcGBwwYUOBiAADIr3ydQrwy6/CGG7PZdOCA+afvOIWI4o5TiCjunHYKMSEh4aaLAQDAmRyehQgAQFFAgAEALIkAAwBYEgEGALAkAgwAYEn5moW4ffv2fG8wIiKiwMUAAJBf+QqwunXrymazyTAM2Wy26/bNzMx0SmEAAFxPvk4hJiQk6MCBA0pISNDChQtVsWJFTZ8+XVu3btXWrVs1ffp0Va5cWQsXLizsegEAkJTPI7Dy5cvbXz/zzDOKiYnR448/bm+LiIhQWFiYRowYoTZt2ji9SAAA/s7hSRw7duzI89ZSFStW1K5du5xSFAAAN+JwgNWoUUNvv/22Ll++bG9LTU3V22+/rRo1aji1OAAArsXhx6nMmDFDTz75pMLCwuzP/9q2bZtsNpu+++47pxcIAEBeHA6w+++/XwkJCZo3b552794twzDUoUMHderUSb6+voVRIwAAuRTogZY+Pj7q06ePs2sBACDfCnQnjs8//1wPPvigypQpo0OHDkmSJk+erH//+99OLQ4AgGtxOMA++ugjvfLKK2rZsqXOnDljv3C5ZMmSmjJlirPrAwAgT/l6IvPVwsPD9e6776pNmzby9/fXtm3bVKlSJf32229q2rSpEhMTC6vWfHPzKGt2CUChahl6j9klAIVq6eEbTwp0+AgsISFB99yT+z8eT09PpaSkOLo5AAAKxOEAq1ixouLi4nK1//DDDwoPD3dGTQAA3JDDsxCHDBmiF198UZcvX5ZhGNq4caO+/PJLjRs3TrNmzSqMGgEAyMXhAHvuueeUkZGhoUOH6uLFi+rUqZPKli2rqVOnqmPHjoVRIwAAuTg8ieNqiYmJysrKUkhIiDNrumlM4kBxxyQOFHeFMomjWbNmOnv2rCQpKCjIHl7nzp1Ts2bNHN0cAAAF4nCArVy5UmlpabnaL1++rDVr1jilKAAAbiTfY2Dbt2+3v961a5dOnjxpf5+Zmally5apbFlO3QEAbo18B1jdunVls9lks9nyPFXo7e2tadOmObU4AACuJd8BlpCQIMMwVKlSJW3cuFHBwcH2ZR4eHgoJCZGrq2uhFAkAwN/lO8DKly8vScrKyiq0YgAAyC+HJ3GMGzdOn332Wa72zz77TBMmTHBKUQAA3IjDAfbxxx+revXqudpr1qypGTNmOKUoAABuxOEAO3nypO68885c7cHBwTpx4oRTigIA4EYcDrCwsDCtW7cuV/u6detUpkwZpxQFAMCNOHwvxF69eunll19Wenq6fTr98uXLNXToUA0ePNjpBQIAkBeHA2zo0KFKSkrSCy+8YL8jh5eXl4YNG6bhw4c7vUAAAPJS4Jv5XrhwQfHx8fL29lbVqlXl6enp7NoKjJv5orjjZr4o7vJzM1+Hj8Cu8PPz0z/+8Y+Crg4AwE3JV4C1a9dOsbGxKlGihNq1a3fdvosWLXJKYQAAXE++AiwgIEA2m83+GgAAs93UAy2LKsbAUNwxBobirlAeaAkAQFGQr1OI99xzj/0U4o1s2bLlpgoCACA/8hVgbdq0sb++fPmypk+frvDwcDVo0ECS9Msvv2jnzp164YUXCqVIAAD+Ll8BNmrUKPvrXr16acCAARo7dmyuPkeOHHFudQAAXIPDkzgCAgK0adMmVa1aNUf7vn37VK9ePSUnJzu1wIJgEgeKOyZxoLgrlEkc3t7eWrt2ba72tWvXysvLy9HNAQBQIA7fiePll19Wv379tHnzZtWvX19S9hjYZ599ppEjRzq9QAAA8uJwgL322muqVKmSpk6dqvnz50uSatSoodjYWLVv397pBQIAkBcuZAYsiDEwFHeFdiHz2bNnNWvWLL3++utKSkqSlH3917FjxwqyOQAAHObwKcTt27erefPmCggI0MGDB9WrVy+VKlVKixcv1qFDhzR37tzCqBMAgBwcPgJ75ZVX1KNHD+3bty/HrMOWLVtq9erVTi0OAIBrcTjAfv31V/Xt2zdXe9myZXXy5EmnFAUAwI04HGBeXl46d+5crvY9e/YoODjYKUUBAHAjDgdY69at9dZbbyk9PV2SZLPZdPjwYb322mt66qmnnF4gAAB5cTjAJk2apNOnTyskJESXLl1SkyZNVKVKFfn7++udd94pjBoBAMjF4VmIJUqU0Nq1a7VixQpt2bJFWVlZuvfee9W8efPCqA8AgDw5FGAZGRny8vJSXFycmjVrpmbNmhVWXQAAXJdDpxDd3NxUvnx5ZWZmFlY9AADki8NjYG+++aaGDx9uvwMHAABmcHgMLCYmRr///rvKlCmj8uXLy9fXN8fyLVu2OK04AACuxeEAa926tWw2W2HUAgBAvnE3esCCuBs9ijun3o3+4sWLevHFF1W2bFmFhISoU6dOSkxMvKkCAQAoqHwH2KhRoxQbG6snnnhCHTt21E8//aR+/foVZm0AAFxTvsfAFi1apE8//VQdO3aUJHXp0kWRkZHKzMyUq6troRUIAEBe8n0EduTIETVq1Mj+/v7775ebm5uOHz9eKIUBAHA9+Q6wzMxMeXh45Ghzc3NTRkaG04sCAOBG8n0K0TAM9ejRQ56enva2y5cv6/nnn89xLdiiRYucWyEAAHnId4B17949V1uXLl2cWgwAAPmV7wCbPXt2YdYBAIBDHL4XIgAARQEBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWZHqAJSQkaN++fbna9+3bp4MHD976ggAAlmB6gPXo0UPr16/P1b5hwwb16NHj1hcEALAE0wNs69atioyMzNVev359xcXF3fqCAACWYHqA2Ww2nT9/Pld7cnKyMjMzTagIAGAFpgdYo0aNNG7cuBxhlZmZqXHjxunBBx80sTIAQFHmZnYB0dHRaty4sapVq6ZGjRpJktasWaNz585pxYoVJlcHACiqTD8CCw8P1/bt29W+fXudOnVK58+fV7du3bR7927VqlXL7PIAAEWUzTAMw+winM3No6zZJQCFqmXoPWaXABSqpYe/u2EfU04hbt++XbVq1ZKLi4u2b99+3b4RERG3qCoAgJWYEmB169bVyZMnFRISorp168pmsymvA0GbzcZMRABAnkwJsISEBAUHB9tfo+jr26eb+vbtqgrlwyRJu3bt1dvvTNayH3+WJH06a7K6d2ufY50NG7YostGTt7xW4O+eHdRJnQZ1ytF25tQZdavXVZIUGBSoHsN7qG7je+RXwle/bdipj0d+rBMHj9v7vzjuRdV5sK5K3VFKl1MuK35zvOaMi9XR/Uftfdr3b696zf6hSjUrKj0tQ8/W7nhrPuBtypQAK1++fJ6v/64YDs9Z1rFjJ/TGG+P0+/6DkqRuXZ/RooWfqd79LbRr115J0rJlK9Sz9yv2ddLS0s0oFcjToT2H9GanN+zvszKz7K/fmPmmMjIy9E7Pt3XxwkW16d1Gb89/Wy883E+pl1IlSb/v+F0rF6/U6eOn5R/or2cHddJb895Sr8heysrK3pabh5vW/Wetdm/ZrUc6PHJrP+BtyPRZiF27dtWFCxdytR88eFCNGzc2oSLk5bv//KQflq3Qvn0HtG/fAY0YOUEXLqTogfvvtfdJTUvTH3+ctv+cOXPWvIKBv8nMyNTZ02ftP+eSzkmSylQso+r3VddHb0zXvu37dOzAMX30xkfy8vVSk9ZN7Ov/OP9H7dy4U6eOntL+3/Zr3sTPFVw2RCFhIfY+89+fr39/+m8d2n3wVn+825LpAbZr1y7Vrl1b69ats7fNmTNHderU0R133GFiZbgWFxcXtW/fSr6+Pvplw2Z7e5PGDXT86Dbt2rlGMz6KVnBwaROrBHIqU7GMYn+do1lrZ2nIB0N1R7nsf1/cPdwlSWmpafa+WVlZykjPUPg/wvPclqe3p5q3b66Th08q8Xhi4RePPJl+IfOGDRv05ptvqlmzZho8eLD27dunZcuWaerUqYqKirrh+qmpqUpNTc3RZhiGbDZbYZV826pVq7rWrv5WXl6eunAhRU8/00vx8dlPElj2489auPA7HTp8VBUrlNPo0UP00//7Svc/0FJpaWk32DJQuPZu3aPJg97XsQPHFBgcqA4vddTERZP0YvMXdHT/Uf1x5A91H9ZdHwz/QKkXU9WmdxuVCimlkiGlcmzn8a6Pq8frz8nb11tH9h3RiM5vKiM9w6RPhSJzHdioUaM0duxYubm5adWqVWrQoEG+1hs9erTGjBmTo83m4icX1xKFUeZtzd3dXeXKlVVgQAm1a/e4op7rpGbNn7KH2NVCQ0N04PcN6tTlBS1Z8oMJ1RZvXAd2czy9PTVzzSwtnLFQ/561RJVrV9aA6IGqVLOSMjMyFbc2TkZW9j+NY3qMtq/n4++jgNKBKhVSUm37tlPp0NIa2m6I0lNzjvc+/PTD6jWqN5M4bkJ+rgMz/RRienq6Bg8erAkTJmj48OFq0KCB2rZtq++//z5f6w8fPlzJyck5fmwu/oVc9e0pPT1d+/cf1OYt2/XGm+O1ffsuvdS/V559T548pUOHjqlqlYq3uErgxlIvpergnoMqU7GMJGn/jv0a2HKAOtRsr271ump0t1HyL+mvP46czLHexfMXdeLgce3cuFPjnx+nuyrfpQYt8vfHNpzP9FOI9erV08WLF7Vy5UrVr19fhmEoOjpa7dq1U1RUlKZPn37d9T09PeXp6ZmjjdOHt4bNZpOnp0eey0qVKqmwsDt14uSpW1wVcGNuHm4KqxKmXRt35mi/eP6iJOnOCmVUJaKKvpg077rbsdn+N4aGW69IBFhMTIx8fX0lZf+jOGzYMLVo0UJdunQxuTpc8fbY17Rs2QodOXpc/v5+6tC+tZo0aaAn/tlZvr4+GjVisBYt/l4nTv6hCuXD9PbY15SYeIbThygSot6I0sb/btTp46cVUDpAHQZ0lI+fj5Z/s1ySFPlEpJL/PKfTx0+pQrUK6j26jzb8+Iu2rtkqSbqj3B1q9GRjbV29Ref+PKdSoaX1dL+nlHo5TZt+3mTfT3CZYPkF+im4bLBcXF1UMTz7DMSJgyd0+eLlW//BiznTA+zTTz/Ns71u3bravHlznstw64WEBCl2dozuvDNEycnntWNHvJ74Z2f9d/kaeXl5qVat6urS5WkFBpbQiROntHLVej3buZ8uXEgxu3RApe8M0qsfDFGJkiV0Lumc9mzZrVfbDNbpY6clSaVCSqnniF4KDArUmVNntGLhCi2I+Zd9/fTUdNX8R021imolvwA/nU08q50bdmpo2yFK/jPZ3q/z4M56+Jnm9vcxy6ZJkoa3H67fftlxiz7t7aNITOJYtWqVJk2apPj4eNlsNtWoUUNDhgyxP17FUdzMF8UdkzhQ3FliEse8efPUvHlz+fj4aMCAAerfv7+8vb318MMPa/78+WaXBwAookw/AqtRo4b69OmjQYMG5Wh///33NXPmTMXHxzu8TY7AUNxxBIbizhJHYAcOHNCTT+a+4WurVq240S8A4JpMD7CwsDAtX748V/vy5csVFhZmQkUAACswbRZiVFSUpk6dqsGDB2vAgAGKi4tTw4YNZbPZtHbtWsXGxmrq1KlmlQcAKOJMGwNzdXXViRMnFBISosWLF+u9996zj3ddmYXYunXrAm2bMTAUd4yBobjLzxiYaUdgV+dm27Zt1bZtW7NKAQBYkKljYNzyCQBQUKbeiePuu+++YYglJSXdomoAAFZiaoCNGTNGAQEBZpYAALAoUwOsY8eOCgkJuXFHAAD+xrQxMMa/AAA3w7QAKwL3EAYAWJhppxCzsrLM2jUAoBgw/VZSAAAUBAEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYks0wDMPsImBtqampGjdunIYPHy5PT0+zywGciu930UWA4aadO3dOAQEBSk5OVokSJcwuB3Aqvt9FF6cQAQCWRIABACyJAAMAWBIBhpvm6empUaNGMcCNYonvd9HFJA4AgCVxBAYAsCQCDABgSQQYAMCSCDAUSbGxsQoMDDS7DOC6Dh48KJvNpri4OLNLuS0RYMVIjx49ZLPZNH78+BztS5Yskc1mc2hbFSpU0JQpU665fOXKlbLZbNf9iY2NLcCnyNahQwft3bu3wOsDV9zoe9qjR48CbzssLEwnTpxQrVq1nFcw8s3N7ALgXF5eXpowYYL69u2rkiVLFtp+GjZsqBMnTtjfDxw4UOfOndPs2bPtbQEBAQXevre3t7y9vW+qRkBSju/pggULNHLkSO3Zs8fedjPfM1dXV4WGht5UfSg4jsCKmebNmys0NFTjxo27br+FCxeqZs2a8vT0VIUKFfTee+/ZlzVt2lSHDh3SoEGD7H+l/p2Hh4dCQ0PtP97e3vL09LS/L1mypIYNG6aQkBB5eXnpwQcf1K+//ipJunz5smrWrKk+ffrYt5eQkKCAgADNnDlTUt6nEL/99lvVq1dPXl5eCgoKUrt27Qr6a8Jt5OrvaUBAgGw2W462+fPnq3LlyvLw8FC1atX0+eef29eNiopSRESEUlNTJUnp6em677771LlzZ0l5n0LcuXOnnnjiCZUoUUL+/v5q1KiR9u/ff0s/8+2CACtmXF1d9e6772ratGk6evRonn02b96s9u3bq2PHjtqxY4dGjx6tESNG2E/5LVq0SHfddZfeeustnThxIsdfsPk1dOhQLVy4UHPmzNGWLVtUpUoVtWjRQklJSfLy8tIXX3yhOXPmaMmSJcrMzFTXrl310EMPqXfv3nlu7z//+Y/atWunJ554Qlu3btXy5ctVr149h+sCrrZ48WINHDhQgwcP1m+//aa+ffvqueee088//yxJiomJUUpKil577TVJ0ogRI5SYmKjp06fnub1jx46pcePG8vLy0ooVK7R582ZFRUUpIyPjln2m24qBYqN79+5G69atDcMwjPr16xtRUVGGYRjG4sWLjav/r+7UqZPxyCOP5Fh3yJAhRnh4uP19+fLljcmTJxdo3xcuXDDc3d2NL774wr48LS3NKFOmjBEdHW1vi46ONoKCgoyXXnrJCA0NNU6fPm1fNnv2bCMgIMD+vkGDBkbnzp3zXQ+Ql79/rxo2bGj07t07R59nnnnGePzxx+3v169fb7i7uxsjRoww3NzcjFWrVtmXJSQkGJKMrVu3GoZhGMOHDzcqVqxopKWlFernQDaOwIqpCRMmaM6cOdq1a1euZfHx8YqMjMzRFhkZqX379ikzM/Om971//36lp6fn2Ie7u7vuv/9+xcfH29sGDx6satWqadq0aZo9e7aCgoKuuc24uDg9/PDDN10bcLVr/bdw9fe0QYMGevXVVzV27FgNHjxYjRs3vub24uLi1KhRI7m7uxdazfgfAqyYaty4sVq0aKHXX3891zLDMHKNaxlOvKPYlW3ltY+r206dOqU9e/bI1dVV+/btu+42mdCBwnKj72lWVpbWrVvH97QIIsCKsfHjx2vp0qVav359jvbw8HCtXbs2R9v69et19913y9XVVVL2JI2CHo1VqVJFHh4eOfaRnp6uTZs2qUaNGva2qKgo1apVS3PnztXQoUPzPFq8IiIiQsuXLy9QPcC11KhRI8//Fq7+nk6cOFHx8fFatWqVfvzxxxwzbf8uIiJCa9asUXp6eqHVjKuYegITTnX1ONQVXbt2Nby8vHKMgW3evNlwcXEx3nrrLWPPnj1GbGys4e3tbcyePdve55FHHjFatWplHD16NMfYVH73PXDgQKNMmTLGDz/8YOzcudPo3r27UbJkSSMpKckwDMP44IMPjMDAQOPw4cOGYRhGly5djLp16xqpqamGYeQeq/j5558NFxcXY+TIkcauXbuM7du3GxMmTHDwN4Tb3d+/V4sXLzbc3d2Njz76yNi7d6/x3nvvGa6ursbPP/9sGIZhbN261fDw8DC+/fZbwzAMY9asWYa/v7+xf/9+wzByj4ElJiYapUuXNtq1a2f8+uuvxt69e425c+cau3fvvpUf87ZBgBUjeQXYwYMHDU9PT+Pvf6t88803Rnh4uOHu7m6UK1fOmDhxYo7l//d//2dERETkuW5+9n3p0iXjpZdeMoKCggxPT08jMjLS2Lhxo2EYhhEfH294e3sb8+fPt/dPTk42KlSoYAwdOtQwjNz/0BiGYSxcuNCoW7eu4eHhYQQFBRnt2rW7YV3A1fL6Xk2fPt2oVKmS4e7ubtx9993G3LlzDcPI/g6Hh4cbffr0ydG/bdu2RsOGDY2MjIxcAWYYhrFt2zbj0UcfNXx8fAx/f3+jUaNG9sCDc/E4FQCAJTEGBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGWFTTpk318ssv57v/ypUrZbPZdPbs2Zvab4UKFTRlypSb2gbgDAQY4IArT6i+1k+PHj3MLhG4bbiZXQBgJVc/nXrBggUaOXKk9uzZY2/7++M00tPTeTYUUEg4AgMcEBoaav8JCAiQzWazv798+bICAwP11VdfqWnTpvLy8tK8efM0evRo1a1bN8d2pkyZogoVKuRomz17tmrUqCEvLy9Vr179mo+tv5Z58+apXr168vf3V2hoqDp16qRTp07l6rdu3TrVqVNHXl5eeuCBB7Rjx44cy9evX6/GjRvL29tbYWFhGjBggFJSUhyqBbgVCDDAyYYNG6YBAwYoPj5eLVq0yNc6M2fO1BtvvKF33nlH8fHxevfddzVixAjNmTMn3/tNS0vT2LFjtW3bNi1ZskQJCQl5ntIcMmSIJk2apF9//VUhISFq1aqV/flVO3bsUIsWLdSuXTtt375dCxYs0Nq1a9W/f/981wHcKpxCBJzs5ZdfVrt27RxaZ+zYsXrvvffs61WsWFG7du3Sxx9/rO7du+drG1FRUfbXlSpVUkxMjO6//35duHBBfn5+9mWjRo3SI488IkmaM2eO7rrrLi1evFjt27fXxIkT1alTJ/vkkKpVqyomJkZNmjTRRx99JC8vL4c+F1CYCDDAyerVq+dQ/9OnT+vIkSPq2bOnevfubW/PyMhQQEBAvrezdetWjR49WnFxcUpKSlJWVpYk6fDhwwoPD7f3a9Cggf11qVKlVK1aNcXHx0uSNm/erN9//11ffPGFvY9hGMrKylJCQkKOJxUDZiPAACfz9fXN8d7FxUV/f+ze1Y+cvxI0M2fO1AMPPJCjn6ura772mZKSokcffVSPPvqo5s2bp+DgYB0+fFgtWrRQWlraDde32Wz2Wvr27asBAwbk6lOuXLl81QLcKgQYUMiCg4N18uRJGYZhD4q4uDj78jvuuENly5bVgQMH1Llz5wLtY/fu3UpMTNT48eMVFhYmSdq0aVOefX/55Rd7GJ05c0Z79+5V9erVJUn33nuvdu7cqSpVqhSoDuBWIsCAQta0aVOdPn1a0dHRevrpp7Vs2TL98MMPKlGihL3P6NGjNWDAAJUoUUItW7ZUamqqNm3apDNnzuiVV1654T7KlSsnDw8PTZs2Tc8//7x+++03jR07Ns++b731lkqXLq077rhDb7zxhoKCgtSmTRtJ2RNQ6tevrxdffFG9e/eWr6+v4uPj9dNPP2natGlO+X0AzsIsRKCQ1ahRQ9OnT9eHH36oOnXqaOPGjXr11Vdz9OnVq5dmzZql2NhY1a5dW02aNFFsbKwqVqyYr30EBwcrNjZWX3/9tcLDwzV+/HhNmjQpz77jx4/XwIEDdd999+nEiRP69ttv5eHhIUmKiIjQqlWrtG/fPjVq1Ej33HOPRowYoTvvvPPmfglAIbAZfz85DwCABXAEBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkv4/7eYoVxN/+fcAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "mat = confusion_matrix(y_train, labels)\n",
        "sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False,\n",
        "             xticklabels=['Not Toxic', 'Toxic'], yticklabels=['Not Toxic', 'Toxic'])\n",
        "plt.xlabel('True label')\n",
        "plt.ylabel('Predicted label')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "lQOxb4hbYFS_"
      },
      "source": [
        "## 5.2 | K-Means using 2 clusters with t-SNE."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OjOoa7sIYFS_"
      },
      "outputs": [],
      "source": [
        "from sklearn.manifold import TSNE\n",
        "\n",
        "# Using t-SNE\n",
        "\n",
        "# Project the data: this step will take several seconds\n",
        "tsne = TSNE(n_components=2, init='random', random_state=69)\n",
        "projection = tsne.fit_transform(X_train_dtm)\n",
        "\n",
        "# Compute the clusters\n",
        "kmeans = KMeans(n_clusters=2, n_init=10, random_state=69)\n",
        "clusters = kmeans.fit_predict(projection)\n",
        "\n",
        "# Permute the labels\n",
        "labels = np.zeros_like(clusters)\n",
        "for i in range(2):\n",
        "    mask = (clusters == i)\n",
        "    labels[mask] = mode(y_train[mask], keepdims=True)[0]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "dHqIido1YFS_"
      },
      "source": [
        "### Check for accuracy."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uazBSi4xYFS_",
        "outputId": "747817d1-453a-49bb-b227-8b357d67cb63"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.7403185568988678"
            ]
          },
          "execution_count": 100,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "accuracy_score(y_train, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ruj6FicLYFS_",
        "outputId": "7a4482bd-215d-41e9-d9ed-ea2c29c928f9"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Text(113.9222222222222, 0.5, 'Predicted label')"
            ]
          },
          "execution_count": 101,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbAAAAGwCAYAAADITjAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyOUlEQVR4nO3de5xN9f7H8feeGXNxmTGMMZHLMGIGg3LknqTo+IVUiFyjVCI5aHQQyl1uB3WQMZGQS+VUJ1GuRS7jOi5l3MdlGpfCjLms3x+Ond0M9h57rFnj9Xw85vHY813f9d2fpa23tdZ3f5fNMAxDAABYjIfZBQAAkB0EGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQvswvICamJh8wuAchRzar3NLsEIEetOv7tbftwBgYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALMn0AHv22Wc1evToTO3jxo3Tc889Z0JFAAArMD3A1qxZo+bNm2dqb9asmdauXWtCRQAAKzA9wP744w95e3tnas+XL58uXrxoQkUAACswPcCqVKmihQsXZmr/9NNPFRERYUJFAAAr8DK7gMGDB+uZZ57Rr7/+qsaNG0uSVq1apQULFmjx4sUmVwcAyK1MD7AWLVpo+fLlGjlypD777DP5+fkpMjJS3333nR555BGzywMA5FKmB5gkNW/ePMuJHAAA3Izp98AAAMgOU87AihQpogMHDigoKEiBgYGy2Ww37ZuUlHQXKwMAWIUpATZx4kQVKlTI/vpWAQYAQFZshmEYZhfhbqmJh8wuAchRzar3NLsEIEetOv7tbfuYfg9s9uzZWbanpaUpKirqLlcDALAK0wOsX79+euaZZxzude3bt0+1atXSokWLTKwMAJCbmR5g27dv1+nTp1W1alWtXLlS06ZN04MPPqgqVaooNjbW7PIAALmU6d8DCw0N1dq1a9W3b181a9ZMnp6eiomJUbt27cwuDQCQi5l+BiZJK1as0IIFC1S3bl0VLlxYM2fO1MmTJ80uCwCQi5keYC+//LLatGmjAQMGaO3atdq5c6d8fHxUtWpV7oEBAG7K9EuIGzZs0KZNm1StWjVJUkhIiL766itNmzZN3bp1U5s2bUyuEACQG5keYFu3bpWPj0+m9tdee01NmjQxoSIAgBWYHmA+Pj46f/68Zs+erbi4ONlsNoWHh+vFF19UxYoVzS4PAJBLmX4PbMuWLSpfvrwmTpyopKQkJSYmauLEiSpfvry2bdtmdnkAgFzK9KWkGjRooLCwMM2cOVNeXtdOCNPS0tS9e3cdOnRIa9eudXlMlpJCXsdSUsjrnFlKyvRLiFu2bHEIL0ny8vLSgAEDVLNmTRMrAwDkZqYHmL+/v44ePapKlSo5tB87dsy+Yj3cZ2bMQn23ZoPijxyXr4+3qleNUN9Xuim0zP1Z9h82dooWf/61BvZ+SR3bPm1vX/z5V/rPyh8Ut/8XXbp8RRu/WSz/QgUd9j189LgmTJut7bv2KjU1VRXKh6p3j06q9dC1GafL/7NS/xz5fpbvu2bFAhUNLOyeg8Y9z8PTQ53f7KTHnm6sIsGB+u10kr5d/K3mTf5EWV2E6ju6j/7vheaaNnSGls5e5tD+YP0aKhpSVFcuXdGeLXs1c+RsHfv1mL1P+9efV+3Haql85fJKu5qmlpVb35VjvBeZFmAxMTFq27at2rZtqxdffFHjx49X3bp1ZbPZtH79evXv31/PP/+8WeXlWVtid+n51k+pSvgDSktP15R/z9VLfd/W5/M/VH4/X4e+q9Zu1M49+xUcVDTTOMnJKar/cE3Vf7imJn0wJ8v3erX/UJUpVVKzp4yWr4+3Pl60XK8NGKqvF32koKJF1KxJQ9Wv/ZDDPm+/975Srl4lvOBW7V5tq6c6NteYN8bp8IEjqljtAfWf0E+Xfr+kpbOXO/St17SuKtWopMRTiZnGObDroL5btlpnTpyRf+FC6vRmR435ZJReqNNJGRkZkqR83l5as2Kd9m6N05Ptmt2Nw7tnmRZgXbt2VbNmzTR+/HjZbDZ16tRJaWlpkqR8+fLplVde0ejRo80qL8/68P13HX5/d1BfNfy/57V3/0HVrF7V3n76bKJGvj9dH77/nl7tPyTTONfPxjZv25nl+5w7f0FHj5/UiKi+qhgWKknq27OrPl26Qr/EH1FQ0SLy9fGR7w1foUg6d16btu7Q8Kg37vQwAQeVHwrXxm9/1KbVmyVJp4+f1qMtG+mByAcc+gWFFNXr776mgR0GaeTcEZnG+c/8r+yvTx8/rTnjojVz5YcqXqq4Eo4kSJLmTvhYktT0ucdz6nDwP6bNQrx+2u7t7a3Jkyfr3Llzio2N1fbt25WUlKSJEydm+f0wuNcfly5LkgL8/7xcm5GRoajh49Wl/bMKK1cmW+MWDvBXubKl9MU3q3T5SrLS0tK16POvVLRIoCIqVshyny++WSU/Xx898Wj9bL0ncDO7ft6jGvWq6/7QkpKkcuHlVPVvVeyBJkk2m01vTR6oRR8s1pEDR247pq+fr5q2aaqTRxJ09uTZHKsdN2fqPbAbn8ScP39+Va1a9Ra94W6GYWjslH/rwcjKqlCurL199rzF8vT00AvPtcz22DabTTMnjdTrA4fr4cdby8PDpqKBgfpwwohM98quW7biW/398UYOZ2WAO3w6baEKFCqgOWtmKyM9Qx6eHvpoTLS+//wHe592r7ZVelp6pkuKf9Wi01N66e3u8ivgpyMHj2pA+7eUlpqWsweALJkaYF26dLntWdbSpUtvuT0lJUUpKSkObR4pKZy9OeG996frwK/xipkx3t62Z99BzVv8uRZ/NNXhHxiuMgxD746fpqKBAZo7fZx8fXy05Mtv9NqAofp01hQVCyri0D92d5x+PXxUIwf/I9vvCdzMoy0aqUnrxzSy12gdPnBY5SuX12vvvKLfTv+mbz9bqQpVK6j1i63U88lXbzvWqmWrtHXdVhUJLqo2Lz+rITP+qd5Pv6HUlNS7cCS4kakBVqhQIfn5+d3RGKNGjdKwYcMc2v7Zv7eGDOhzR+PmdSPfn67v1/+kudPGKSS4mL19247dSjp3Xo8/08nelp6eoXH/mqWPFy3Xt0vmOjX+pq2xWrNxszZ+s0gFCxSQJEVU7KUff96uz7/+Tt07Oq5xueTLb1SpQjlVrpT15UXgTrz0zx76dNqn+v6LHyRJ8fsOq3jJ4nq+Vzt9+9lKVa1VRYWDCmvBpvn2fTy9PNVzyEt6pvvT6lDnz78Pl36/rEu/X9aJ+JOK2xan5XuWqn6zeg5nc7g7TA2wKVOmKDg4+I7GiIqK0ptvvunQ5vH7iTsaMy8zDEMj35+hVWs3as6/xuj+EiEO259q9phq/62GQ9vLff+pp5o1Vqu/P+H0+yQnXzsr9rA53mb1sNnss7Wuu3z5iv67ap3e6NnFhSMBnOfr56OMDMfp8hnpGfLwuHaV4bsl32nb+u0O28fMH6mVS77TNwtv/YVam03y9s7n3oLhFNMC7E4uT93Ix8cn0+XC1KuZp7/imncnTNNXK3/QlNFDVCC/nxJ/S5IkFSxYQL4+Pioc4K/CAf4O+3h5eSqoSKDDd8USf0tS4m/ndPT4tee2Hfz1sArk99N9IcEK8C+kalXC5V+ooAa9O0E9u7aXr4+3PvviGx1POK2GdWs5jP/1qrVKT09X8ycezeGjx73qx5U/qUPv53XmxBkdPnBEYVXC9OxLrfXNwv9Kki6e/10Xz//usE9aapqSzpzT8UPHJUn3lQ5Ro6caacvarbrw23kFhQSp3attdTX5qjat/tm+X3CJYipUuJCCSwbLw9ND5SPKSZJOHD6p5MvJd+mI7w2mBZjJK1jdsxYu+48kqWuvgQ7t7w56U62aOz/td+HyrzTjoz8vt3R+rb/DOIGFA/TBhBGa8u+5erH3W0pLS1NYaBlNHT1ElSqUcxhr6Yr/qskjdR1mQgLuNHXwNHXt31l9Rr6uwkGF9dup37Ri3lf6eNI8p8e4mnJVVR+uome6P62CAQV1LvG8dm7apddbvqHzv5239+vyj85q2ubPqxX//vYDSdKbz/1DO37M+msnyB7T1kJcs2aN6tWr57CElLuwFiLyOtZCRF6Xq9dCfOSRR8x6awBAHmD641QAAMgOAgwAYEkEGADAkkwPME9PT505cyZT+2+//SZPT08TKgIAWIHpAXazSZApKSny9va+y9UAAKzCtFmIU6ZMkXTtC82zZs1SwYJ/LvCanp6utWvXZnrIJQAA15kWYBMnTpR07Qzsgw8+cLhc6O3trbJly+qDDz4wqzwAQC5nWoDFx8dLkh599FEtXbpUgYGBZpUCALAgUxfzlaTvv//e/vr6/TB3rZMIAMi7TJ/EIUkxMTGqWrWq/Pz85Ofnp8jISH388cdmlwUAyMVMPwN7//33NXjwYPXq1Uv16tWTYRjasGGDevbsqcTERPXt29fsEgEAuZBpi/leFxoaqmHDhqlTp04O7XPnztU777xjv1fmChbzRV7HYr7I65xZzNf0S4gJCQmqW7dupva6desqISHBhIoAAFZgeoCFhYVp0aJFmdoXLlyoChV4vDwAIGum3wMbNmyY2rZtq7Vr16pevXqy2Wxav369Vq1alWWwAQAg5YIzsGeeeUabNm1SUFCQli9frqVLlyooKEibN2/W008/bXZ5AIBcyvQzMEl66KGHNG+e84/2BgDA9DMwAACyw7QzMA8Pj9uuuGGz2ZSWlnaXKgIAWIlpAbZs2bKbbtu4caOmTp1600etAABgWoC1bNkyU9u+ffsUFRWlL7/8Uh06dNCIESNMqAwAYAW54h7YyZMn1aNHD0VGRiotLU2xsbGaO3euSpcubXZpAIBcytQAu3DhggYOHKiwsDDt2bNHq1at0pdffqkqVaqYWRYAwAJMu4Q4duxYjRkzRiEhIVqwYEGWlxQBALgZ0xbz9fDwkJ+fn5o0aeLwNOa/Wrp0qctjs5gv8joW80Ve58xivqadgXXq1IkHVwIAss20AIuOjjbrrQEAeUCumIUIAICrCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJTm1lNQXX3zh9IAtWrTIdjEAADjLqQBr1aqVU4PZbDalp6ffST0AADjFqQDLyMjI6ToAAHDJHd0DS05OdlcdAAC4xOUAS09P14gRI1SyZEkVLFhQhw5de3jk4MGDNXv2bLcXCABAVlwOsPfee0/R0dEaO3asvL297e1Vq1bVrFmz3FocAAA343KAxcTE6N///rc6dOggT09Pe3tkZKT27dvn1uIAALgZlwPsxIkTCgsLy9SekZGh1NRUtxQFAMDtuBxglStX1rp16zK1L168WDVq1HBLUQAA3I5T0+hvNHToUHXs2FEnTpxQRkaGli5dqv379ysmJkYrVqzIiRoBAMjE5TOwp556SgsXLtRXX30lm82mIUOGKC4uTl9++aUef/zxnKgRAIBMXD4Dk6SmTZuqadOm7q4FAACnZSvAJGnLli2Ki4uTzWZTeHi4HnroIXfWBQDALbkcYMePH9fzzz+vDRs2qHDhwpKk8+fPq27dulqwYIFKlSrl7hoBAMjE5Xtg3bp1U2pqquLi4pSUlKSkpCTFxcXJMAy9+OKLOVEjAACZuHwGtm7dOm3cuFEVK1a0t1WsWFFTp05VvXr13FocAAA34/IZWOnSpbP8wnJaWppKlizplqIAALgdlwNs7Nixev3117VlyxYZhiHp2oSOPn36aPz48W4vEACArNiM6yl0C4GBgbLZbPbfL126pLS0NHl5XbsCef11gQIFlJSUlHPVOik18ZDZJQA5qln1nmaXAOSoVce/vW0fp+6BTZo06U5rAQDArZwKsM6dO+d0HQAAuCTbX2SWpCtXrmSa0OHv739HBQEA4AyXJ3FcunRJvXr1UnBwsAoWLKjAwECHHwAA7gaXA2zAgAFavXq1pk+fLh8fH82aNUvDhg1TiRIlFBMTkxM1AgCQicuXEL/88kvFxMSoUaNG6tatmxo0aKCwsDCVKVNG8+fPV4cOHXKiTgAAHLh8BpaUlKTQ0FBJ1+53XZ82X79+fa1du9a91QEAcBMuB1i5cuV0+PBhSVJERIQWLVok6dqZ2fXFfQEAyGkuB1jXrl21Y8cOSVJUVJT9Xljfvn3Vv39/txcIAEBWnFqJ41aOHj2qLVu2qHz58qpWrZq76rojrMSBvI6VOJDXuW0ljlspXbq0SpcufafDAADgEqcCbMqUKU4P2Lt372wXAwCAs5y6hHh91uFtB7PZdOiQ+ZfvuISIvI5LiMjr3HYJMT4+/o6LAQDAnVyehQgAQG5AgAEALIkAAwBYEgEGALAkAgwAYElOzULcuXOn0wNGRkZmuxgAAJzlVIBVr15dNptNhmHIZrPdsm96erpbCgMA4FacuoQYHx+vQ4cOKT4+XkuWLFFoaKimT5+u7du3a/v27Zo+fbrKly+vJUuW5HS9AABIcvIMrEyZMvbXzz33nKZMmaK///3v9rbIyEiVKlVKgwcPVqtWrdxeJAAAf+XyJI5du3ZlubRUaGio9u7d65aiAAC4HZcDLDw8XO+++66Sk5PtbSkpKXr33XcVHh7u1uIAALgZlx+n8sEHH+ipp55SqVKl7M//2rFjh2w2m1asWOH2AgEAyIrLAVarVi3Fx8dr3rx52rdvnwzDUNu2bdW+fXsVKFAgJ2oEACCTbD3QMn/+/HrppZfcXQsAAE7L1kocH3/8serXr68SJUroyJEjkqSJEyfq888/d2txAADcjMsBNmPGDL355pt68sknde7cOfsXlwMDAzVp0iR31wcAQJaceiLzjSIiIjRy5Ei1atVKhQoV0o4dO1SuXDnt3r1bjRo1UmJiYk7V6jQv75JmlwDkqLVFa5tdApCj6ibcfmEMl8/A4uPjVaNGjUztPj4+unTpkqvDAQCQLS4HWGhoqGJjYzO1f/3114qIiHBHTQAA3JbLsxD79++v1157TcnJyTIMQ5s3b9aCBQs0atQozZo1KydqBAAgE5cDrGvXrkpLS9OAAQN0+fJltW/fXiVLltTkyZPVrl27nKgRAIBMXJ7EcaPExERlZGQoODjYnTXdMSZxIK9jEgfyuhyZxNG4cWOdP39ekhQUFGQPr4sXL6px48auDgcAQLa4HGA//PCDrl69mqk9OTlZ69atc0tRAADcjtP3wHbu3Gl/vXfvXp06dcr+e3p6ur755huVLMmlOwDA3eF0gFWvXl02m002my3LS4V+fn6aOnWqW4sDAOBmnA6w+Ph4GYahcuXKafPmzSpWrJh9m7e3t4KDg+Xp6ZkjRQIA8FdOB1iZMmUkSRkZGTlWDAAAznJ5EseoUaP00UcfZWr/6KOPNGbMGLcUBQDA7bgcYB9++KEqVaqUqb1y5cr64IMP3FIUAAC343KAnTp1Svfdd1+m9mLFiikhIcEtRQEAcDsuB1ipUqW0YcOGTO0bNmxQiRIl3FIUAAC34/JaiN27d9cbb7yh1NRU+3T6VatWacCAAerXr5/bCwQAICsuB9iAAQOUlJSkV1991b4ih6+vrwYOHKioqCi3FwgAQFayvZjvH3/8obi4OPn5+alChQry8fFxd23ZxmK+yOtYzBd5nTOL+bp8BnZdwYIF9be//S27uwMAcEecCrDWrVsrOjpa/v7+at269S37Ll261C2FAQBwK04FWEBAgGw2m/01AABmu6MHWuZW3ANDXsc9MOR1OfJASwAAcgOnLiHWqFHDfgnxdrZt23ZHBQEA4AynAqxVq1b218nJyZo+fboiIiJUp04dSdJPP/2kPXv26NVXX82RIgEA+CunAmzo0KH21927d1fv3r01YsSITH2OHTvm3uoAALgJlydxBAQEaMuWLapQoYJD+8GDB1WzZk1duHDBrQVmB5M4kNcxiQN5XY5M4vDz89P69eszta9fv16+vr6uDgcAQLa4vBLHG2+8oVdeeUVbt25V7drX/hX4008/6aOPPtKQIUPcXiAAAFlxOcDeeustlStXTpMnT9Ynn3wiSQoPD1d0dLTatGnj9gIBAMgKX2QGLIh7YMjrcuyLzOfPn9esWbM0aNAgJSUlSbr2/a8TJ05kZzgAAFzm8iXEnTt3qkmTJgoICNDhw4fVvXt3FSlSRMuWLdORI0cUExOTE3UCAODA5TOwN998U126dNHBgwcdZh0++eSTWrt2rVuLAwDgZlwOsJ9//lkvv/xypvaSJUvq1KlTbikKAIDbcTnAfH19dfHixUzt+/fvV7FixdxSFAAAt+NygLVs2VLDhw9XamqqJMlms+no0aN666239Mwzz7i9QAAAsuJygI0fP15nz55VcHCwrly5okceeURhYWEqVKiQ3nvvvZyoEQCATFyehejv76/169dr9erV2rZtmzIyMvTggw+qSZMmOVEfAABZcinA0tLS5Ovrq9jYWDVu3FiNGzfOqboAALglly4henl5qUyZMkpPT8+pegAAcIrL98D++c9/Kioqyr4CBwAAZnD5HtiUKVP0yy+/qESJEipTpowKFCjgsH3btm1uKw4AgJtxOcBatmwpm82WE7UAAOA0VqMHLIjV6JHXuXU1+suXL+u1115TyZIlFRwcrPbt2ysxMfGOCgQAILucDrChQ4cqOjpazZs3V7t27bRy5Uq98sorOVkbAAA35fQ9sKVLl2r27Nlq166dJOmFF15QvXr1lJ6eLk9PzxwrEACArDh9Bnbs2DE1aNDA/nutWrXk5eWlkydP5khhAADcitMBlp6eLm9vb4c2Ly8vpaWlub0oAABux+lLiIZhqEuXLvLx8bG3JScnq2fPng7fBVu6dKl7KwQAIAtOB1jnzp0ztb3wwgtuLQYAAGc5HWBz5szJyToAAHCJy2shAgCQGxBgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJpgdYfHy8Dh48mKn94MGDOnz48N0vCABgCaYHWJcuXbRx48ZM7Zs2bVKXLl3ufkEAAEswPcC2b9+uevXqZWqvXbu2YmNj735BAABLMD3AbDabfv/990ztFy5cUHp6ugkVAQCswPQAa9CggUaNGuUQVunp6Ro1apTq169vYmUAgNzMy+wCxo4dq4YNG6pixYpq0KCBJGndunW6ePGiVq9ebXJ1AIDcyvQzsIiICO3cuVNt2rTRmTNn9Pvvv6tTp07at2+fqlSpYnZ5AIBcymYYhmF2Ee7m5V3S7BKAHLW2aG2zSwByVN2EJbftY8olxJ07d6pKlSry8PDQzp07b9k3MjLyLlUFALASUwKsevXqOnXqlIKDg1W9enXZbDZldSJos9mYiQgAyJIp98Di4+NVrFgx++tDhw4pPj4+08+hQ4fMKO+eM2Twm0q7esLh5/jR7fbtrVo9qa9WzNepk7uUdvWEqlWrnGkMb29vTZo4QqdO7tKFcwe1bOkclSx5n0OfqLd6a92az3Xx/C9KPLM3x48L9y7/2hGqNDdKNbfPVN2EJSrSrNZN+5Yb+7LqJizRfT2aZ2p/8MdpevjQJ/rb7o9Uac5A+YU53p4oUDVUEZ8OUa19MfrbnmiVG9dTHvl9HfqUHdFNkf8dq9qHP1W1lePdd5AwJ8DKlCkjm81mf32zn9KlS5tR3j1p9559Klmquv2n+oOP2bcVKJBfG3/8WYPeHnnT/d+fMEytWj6pDi+8qkcebaWCBQro8+Vz5eHx50fM2zufPluyQh9+GJOjxwJ45PfRpb2HdejtWbfsV6RZLRWqUUEpCb9l2nZp5yH90neaYhv20d7nR0g2myI+HSz97zOdr3igIhYOVfLhU9rZ/C3FtR+h/A+UUoXJvRzGsUk6s2C1Er/Y4LbjwzWmT6Pv2LGjZsyYoYIFCzq0Hz58WB07dtS6detMquzekpaWrtOnz2a5bf78azdTy5S5P8vt/v6F1K1rO3Xu2kerVl/779Wpy+s6fOhnNXmsgb5duUaSNGz4hGvbOrZxd/mAg/Ort+v86u237OMdUkSh73XX3udHKHzeoEzbT89baX+dcvysjo5ZoOqr35dPqWJKOXJaRR6vKSMtXYeiZkr/uwVyaNBMVf9ugnzLhij58ClJUvzgjyRJpYr6q0B4GXcdIpQLptHv3btXVatW1YYNf/7rZO7cuapWrZqKFy9uYmX3lgphoTp6eKsO7v9R8+dNV2io82e/Dz0YKW9vb638X1BJUkLCae3es1916tTMiXKBO2OzqcLU3jo543NdOXDstt09/HwU3O5RJR85rasnr52t2by9ZFxNs4eXJGUkX5UkFaoVnjN1w4HpAbZp0ya1bdtWjRs31qBBg/Tcc8+pV69emjhxoj777LPb7p+SkqKLFy86/OTBbwbkqM2bt6tLtz76+/91UM9XBiikeDGtW/O5ihQJdGr/4iHFlJKSovPnLzi0nzl9VsWLB+dEycAdKdmrlYz0dCXM+s8t+4V0bqqHf5mn2oc+UeFHa2hP22EyUtMkSRfW71a+4MIq8UpL2fJ5yTOggMpEdZAkeRcvnNOHAOWCS4heXl4aPXq0fHx8NGLECHl5eWnNmjWqU6eOU/uPGjVKw4YNc2izeRSUzdM/J8rNk7757/f217u1Tz/+tEUH9m1Up47PadLkf2d73JvNLgXMVCCynO7r3lw7nuh/275nl67T+bU75V08UCV6tlDFf/fTrhZvy0hJ1ZUDx/RLn6kq+04XlRnUQUZ6hhJmf6WrZ87JSM+4C0cC08/AUlNT1a9fP40ZM0ZRUVGqU6eOnn76aX311VdO7R8VFaULFy44/Ng8CuVw1Xnb5ctXtHv3PoWFhTrV//Sps/Lx8VHhwgEO7cWCg3TmTNb31QCz+D8crnxBAaq55UPVObZIdY4tkm+pYJUd2lkPbp7h0Df998tKjk/QxZ/2an+P8fILK6miTz5s3564bL22VOuuLTV6aHNEFx0bv1D5ivor5eiZu31Y9yTTz8Bq1qypy5cv64cfflDt2rVlGIbGjh2r1q1bq1u3bpo+ffot9/fx8ZGPj49D2/UZjsgeb29vVapUQes3bHKq/9ZtO3X16lU1adJQn332pSQpJCRYVSpXVFTUuzlZKuCys5+t0YW1jgsohC8YrLOfrdWZhbdZf9Vmk807X6bm1MRrl8+D2zVWRkqqzq/d4bZ6cXO5IsCmTJmiAgUKSLoWPgMHDlTTpk31wgsvmFzdvWHs6MFa8Z+VOnrshIKLBWnQoD7y9y+omI8XS5ICAwurdOmSKnHftUk1DzxQXpJ06tQZnT59Vhcv/q6P5nyqcWOGKOm3c0o6d05jRw/Rrt379N2qP2eRlipVQkWKBKp06RLy9PS0f5/sl1/idenS5bt81MjLPPL7yjc0xP67T+lg5a9cVmnn/9DVE4lKO/eHQ38jLV2pZ88p+deT/+tfXEEt6+r8mh1K/e2ivEOKqGSvp5Vx5arOr9pq3y+k65P6fcs+pV9KVuGG1VRmSCcdeW+e0i/++Xn2LRsijwK+yhdcWB6+3spfuawk6cqB4/b7acieXL0WYkpKSqazK2ewFqJr5s+brgb1H1ZQUBGdPfubNm3epqHvjFNc3EFJ16a9fzR7Yqb9ho+YoOEj3pd07Ux4zOh/6vl2T8vPz1erv1+vXq8P0vHjJ+39Z8+aqM6dMk+hf6zJs1qz9sccOrq8ibUQb82/TmVVWTo8U/uZhd/rlzf+lan9wc0zlDBzhRJmXpvUka94oMImvKoCkeXkFVBAqWcv6OKmvTr2/mJ7yElS2JTXFfjYQ/Is4Ksrv5zQyQ++0NnP1jiMXXnJMAXUzbww+da/9VTKcS6x34wzayHmigBbs2aNxo8fr7i4ONlsNoWHh6t///72x6u4igBDXkeAIa9zJsBMn8Qxb948NWnSRPnz51fv3r3Vq1cv+fn56bHHHtMnn3xidnkAgFzK9DOw8PBwvfTSS+rbt69D+/vvv6+ZM2cqLi7O5TE5A0NexxkY8jpLnIEdOnRITz31VKb2Fi1aKD4+3oSKAABWYHqAlSpVSqtWrcrUvmrVKpUqVcqEigAAVmDaNPpu3bpp8uTJ6tevn3r37q3Y2FjVrVtXNptN69evV3R0tCZPnmxWeQCAXM60e2Cenp5KSEhQcHCwli1bpgkTJtjvd12fhdiyZctsjc09MOR13ANDXperp9F7eHjYn8rsbgQY8joCDHldrp/EwZJPAIDsMnUpqQceeOC2IZaUlHSXqgEAWImpATZs2DAFBATcviMAAH9haoC1a9cuR+6BAQDyPtPugXH/CwBwJ0wLsFywhjAAwMJMu4SYkcEjtwEA2Wf6UlIAAGQHAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBgAwJIIMACAJRFgAABLIsAAAJZEgAEALIkAAwBYEgEGALAkAgwAYEkEGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFiSzTAMw+wiYG0pKSkaNWqUoqKi5OPjY3Y5gFvx+c69CDDcsYsXLyogIEAXLlyQv7+/2eUAbsXnO/fiEiIAwJIIMACAJRFgAABLIsBwx3x8fDR06FBucCNP4vOdezGJAwBgSZyBAQAsiQADAFgSAQYAsCQCDLlSdHS0ChcubHYZwC0dPnxYNptNsbGxZpdyTyLA8pAuXbrIZrNp9OjRDu3Lly+XzWZzaayyZctq0qRJN93+ww8/yGaz3fInOjo6G0dxTdu2bXXgwIFs7w9cd7vPaZcuXbI9dqlSpZSQkKAqVaq4r2A4zcvsAuBevr6+GjNmjF5++WUFBgbm2PvUrVtXCQkJ9t/79Omjixcvas6cOfa2gICAbI/v5+cnPz+/O6oRkOTwOV24cKGGDBmi/fv329vu5HPm6empkJCQO6oP2ccZWB7TpEkThYSEaNSoUbfst2TJElWuXFk+Pj4qW7asJkyYYN/WqFEjHTlyRH379rX/K/WvvL29FRISYv/x8/OTj4+P/ffAwEANHDhQwcHB8vX1Vf369fXzzz9LkpKTk1W5cmW99NJL9vHi4+MVEBCgmTNnSsr6EuIXX3yhmjVrytfXV0FBQWrdunV2/5hwD7nxcxoQECCbzebQ9sknn6h8+fLy9vZWxYoV9fHHH9v37datmyIjI5WSkiJJSk1N1UMPPaQOHTpIyvoS4p49e9S8eXP5+/urUKFCatCggX799de7esz3CgIsj/H09NTIkSM1depUHT9+PMs+W7duVZs2bdSuXTvt2rVL77zzjgYPHmy/5Ld06VLdf//9Gj58uBISEhz+BeusAQMGaMmSJZo7d662bdumsLAwNW3aVElJSfL19dX8+fM1d+5cLV++XOnp6erYsaMeffRR9ejRI8vx/vOf/6h169Zq3ry5tm/frlWrVqlmzZou1wXcaNmyZerTp4/69eun3bt36+WXX1bXrl31/fffS5KmTJmiS5cu6a233pIkDR48WImJiZo+fXqW4504cUINGzaUr6+vVq9era1bt6pbt25KS0u7a8d0TzGQZ3Tu3Nlo2bKlYRiGUbt2baNbt26GYRjGsmXLjBv/U7dv3954/PHHHfbt37+/ERERYf+9TJkyxsSJE7P13n/88YeRL18+Y/78+fbtV69eNUqUKGGMHTvW3jZ27FgjKCjIeP31142QkBDj7Nmz9m1z5swxAgIC7L/XqVPH6NChg9P1AFn56+eqbt26Ro8ePRz6PPfcc8bf//53++8bN2408uXLZwwePNjw8vIy1qxZY98WHx9vSDK2b99uGIZhREVFGaGhocbVq1dz9DhwDWdgedSYMWM0d+5c7d27N9O2uLg41atXz6GtXr16OnjwoNLT0+/4vX/99VelpqY6vEe+fPlUq1YtxcXF2dv69eunihUraurUqZozZ46CgoJuOmZsbKwee+yxO64NuNHN/i7c+DmtU6eO/vGPf2jEiBHq16+fGjZseNPxYmNj1aBBA+XLly/HasafCLA8qmHDhmratKkGDRqUaZthGJnuaxluXFHs+lhZvceNbWfOnNH+/fvl6empgwcP3nJMJnQgp9zuc5qRkaENGzbwOc2FCLA8bPTo0fryyy+1ceNGh/aIiAitX7/eoW3jxo164IEH5OnpKenaJI3sno2FhYXJ29vb4T1SU1O1ZcsWhYeH29u6deumKlWqKCYmRgMGDMjybPG6yMhIrVq1Klv1ADcTHh6e5d+FGz+n48aNU1xcnNasWaP//ve/DjNt/yoyMlLr1q1TampqjtWMG5h6ARNudeN9qOs6duxo+Pr6OtwD27p1q+Hh4WEMHz7c2L9/vxEdHW34+fkZc+bMsfd5/PHHjRYtWhjHjx93uDfl7Hv36dPHKFGihPH1118be/bsMTp37mwEBgYaSUlJhmEYxr/+9S+jcOHCxtGjRw3DMIwXXnjBqF69upGSkmIYRuZ7Fd9//73h4eFhDBkyxNi7d6+xc+dOY8yYMS7+CeFe99fP1bJly4x8+fIZM2bMMA4cOGBMmDDB8PT0NL7//nvDMAxj+/bthre3t/HFF18YhmEYs2bNMgoVKmT8+uuvhmFkvgeWmJhoFC1a1GjdurXx888/GwcOHDBiYmKMffv23c3DvGcQYHlIVgF2+PBhw8fHx/jrv1U+++wzIyIiwsiXL59RunRpY9y4cQ7bf/zxRyMyMjLLfZ157ytXrhivv/66ERQUZPj4+Bj16tUzNm/ebBiGYcTFxRl+fn7GJ598Yu9/4cIFo2zZssaAAQMMw8j8PxrDMIwlS5YY1atXN7y9vY2goCCjdevWt60LuFFWn6vp06cb5cqVM/Lly2c88MADRkxMjGEY1z7DERERxksvveTQ/+mnnzbq1q1rpKWlZQowwzCMHTt2GE888YSRP39+o1ChQkaDBg3sgQf34nEqAABL4h4YAMCSCDAAgCURYAAASyLAAACWRIABACyJAAMAWBIBBgCwJAIMAGBJBBhgUY0aNdIbb7zhdP8ffvhBNptN58+fv6P3LVu2rCZNmnRHYwDuQIABLrj+hOqb/XTp0sXsEoF7hpfZBQBWcuPTqRcuXKghQ4Zo//799ra/Pk4jNTWVZ0MBOYQzMMAFISEh9p+AgADZbDb778nJySpcuLAWLVqkRo0aydfXV/PmzdM777yj6tWrO4wzadIklS1b1qFtzpw5Cg8Pl6+vrypVqnTTx9bfzLx581SzZk0VKlRIISEhat++vc6cOZOp34YNG1StWjX5+vrq4Ycf1q5duxy2b9y4UQ0bNpSfn59KlSql3r1769KlSy7VAtwNBBjgZgMHDlTv3r0VFxenpk2bOrXPzJkz9fbbb+u9995TXFycRo4cqcGDB2vu3LlOv+/Vq1c1YsQI7dixQ8uXL1d8fHyWlzT79++v8ePH6+eff1ZwcLBatGhhf37Vrl271LRpU7Vu3Vo7d+7UwoULtX79evXq1cvpOoC7hUuIgJu98cYbat26tUv7jBgxQhMmTLDvFxoaqr179+rDDz9U586dnRqjW7du9tflypXTlClTVKtWLf3xxx8qWLCgfdvQoUP1+OOPS5Lmzp2r+++/X8uWLVObNm00btw4tW/f3j45pEKFCpoyZYoeeeQRzZgxQ76+vi4dF5CTCDDAzWrWrOlS/7Nnz+rYsWN68cUX1aNHD3t7WlqaAgICnB5n+/bteueddxQbG6ukpCRlZGRIko4ePaqIiAh7vzp16thfFylSRBUrVlRcXJwkaevWrfrll180f/58ex/DMJSRkaH4+HiHJxUDZiPAADcrUKCAw+8eHh7662P3bnzk/PWgmTlzph5++GGHfp6enk6956VLl/TEE0/oiSee0Lx581SsWDEdPXpUTZs21dWrV2+7v81ms9fy8ssvq3fv3pn6lC5d2qlagLuFAANyWLFixXTq1CkZhmEPitjYWPv24sWLq2TJkjp06JA6dOiQrffYt2+fEhMTNXr0aJUqVUqStGXLliz7/vTTT/YwOnfunA4cOKBKlSpJkh588EHt2bNHYWFh2aoDuJsIMCCHNWrUSGfPntXYsWP17LPP6ptvvtHXX38tf39/e5933nlHvXv3lr+/v5588kmlpKRoy5YtOnfunN58883bvkfp0qXl7e2tqVOnqmfPntq9e7dGjBiRZd/hw4eraNGiKl68uN5++20FBQWpVatWkq5NQKldu7Zee+019ejRQwUKFFBcXJxWrlypqVOnuuXPA3AXZiECOSw8PFzTp0/XtGnTVK1aNW3evFn/+Mc/HPp0795ds2bNUnR0tKpWrapHHnlE0dHRCg0Ndeo9ihUrpujoaC1evFgREREaPXq0xo8fn2Xf0aNHq0+fPnrooYeUkJCgL774Qt7e3pKkyMhIrVmzRgcPHlSDBg1Uo0YNDR48WPfdd9+d/SEAOcBm/PXiPAAAFsAZGADAkggwAIAlEWAAAEsiwAAAlkSAAQAsiQADAFgSAQYAsCQCDABgSQQYAMCSCDAAgCURYAAAS/p/V4HLF3rPpqAAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "mat = confusion_matrix(y_train, labels)\n",
        "sns.heatmap(mat.T, square=True, annot=True, fmt='d', cbar=False,\n",
        "             xticklabels=['Not Toxic', 'Toxic'], yticklabels=['Not Toxic', 'Toxic'])\n",
        "plt.xlabel('True label')\n",
        "plt.ylabel('Predicted label')"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "wABSsJf-YFS_"
      },
      "source": [
        "There is a significant increase in accuracy rate when using t-SNE. Moving forward, we will use it to determine the best cluster size."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xnI_-Qf5YFS_"
      },
      "source": [
        "## 5.3 | Try 1-10 clusters while using t-SNE."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IQF2bNEmYFTA",
        "outputId": "5baaa001-9e37-40b1-8038-b230c3125860"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "c:\\Users\\floss\\anaconda3\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3432: RuntimeWarning: Mean of empty slice.\n",
            "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
            "c:\\Users\\floss\\anaconda3\\lib\\site-packages\\numpy\\core\\_methods.py:182: RuntimeWarning: invalid value encountered in divide\n",
            "  ret = um.true_divide(\n"
          ]
        }
      ],
      "source": [
        "kmeans = []\n",
        "accuracy_scores = []\n",
        "inertias = []\n",
        "\n",
        "for i in range(1, 11):\n",
        "    # Project the data: this step will take several seconds\n",
        "    temp_tsne = TSNE(n_components=2, init='random', random_state=69)\n",
        "    temp_projection = temp_tsne.fit_transform(X_train_dtm)\n",
        "\n",
        "    # Compute the clusters\n",
        "    temp_kmeans = KMeans(n_clusters=i, n_init=10, random_state=69)\n",
        "    temp_clusters = temp_kmeans.fit_predict(temp_projection)\n",
        "    kmeans.append(temp_kmeans)\n",
        "\n",
        "    # Permute the labels\n",
        "    temp_labels = np.zeros_like(temp_clusters)\n",
        "    for j in range(2):\n",
        "        mask = (temp_clusters == j)\n",
        "        temp_labels[mask] = mode(y_train[mask], keepdims=True)[0]\n",
        "\n",
        "    # Compute the accuracy\n",
        "    temp_accuracy_score = accuracy_score(y_train, temp_labels)\n",
        "    accuracy_scores.append(temp_accuracy_score)\n",
        "\n",
        "    # Get the inertia\n",
        "    temp_inertia = temp_kmeans.inertia_\n",
        "    inertias.append(temp_inertia)\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "XXV9SFCTYFTA"
      },
      "source": [
        "### Determine the best cluster size."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ow0-fLNQYFTA",
        "outputId": "8ca7d65f-f1cb-40ab-93d7-71a519084b75"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAioAAAHFCAYAAADcytJ5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABLm0lEQVR4nO3deVxU9f4/8NfMADNsMyyyg4AL7rtWLrlLknndLdNSy75WVpo/u+m9lqYpV83ypmXXLKzcSg0rzX3BXVHEfUFFQRYBgRnWAWbO7w9gkkBlEDizvJ6Px3nkHA7Di7g3Xn7OOe8jEQRBABEREZEJkoodgIiIiOhhWFSIiIjIZLGoEBERkcliUSEiIiKTxaJCREREJotFhYiIiEwWiwoRERGZLBYVIiIiMlksKkRERGSyWFSILMyaNWsgkUgeuh08eNBwbFBQECZMmGB4ffDgQUgkEmzevLn+gxvh8OHDGD16NPz8/GBnZweVSoVu3bph5cqVyMvLEzseEdUiG7EDEFHdiIiIQPPmzSvtb9mypQhpas+cOXMwb948dOvWDfPnz0fjxo2Rn5+PY8eOYe7cubh+/Tq++OILsWMSUS1hUSGyUK1bt0bnzp3FjlGrNm3ahHnz5uH111/Ht99+C4lEYvhYWFgY/vnPf+L48eO18rXy8/Ph4OBQK+9FRDXHUz9EVElhYSGmT58Ob29v2Nvbo1evXjh79myl437//Xd07doVDg4OcHZ2xoABAyoUhUuXLkEikWDTpk2GfWfOnIFEIkGrVq0qvNc//vEPdOrU6ZG55s2bB1dXV3z55ZcVSko5Z2dnhIaGAgBu374NiUSCNWvWVDpOIpFg7ty5htdz586FRCJBTEwMRo4cCVdXVzRu3BjLli2DRCLBjRs3Kr3Hhx9+CDs7O2RkZBj27d27F/369YNSqYSDgwO6d++Offv2PfJ7IqJHY1EhslA6nQ4lJSUVNp1OV63P/de//oVbt25h9erVWL16NZKTk9G7d2/cunXLcMz69esxZMgQKJVKbNiwAd999x2ysrLQu3dvHDlyBADQqlUr+Pj4YO/evYbP27t3L+zt7XH58mUkJycDAEpKShAVFYX+/fs/NFNKSgouXryI0NDQOlvpGD58OJo0aYJNmzbhm2++wbhx42BnZ1ep7Oh0OqxduxaDBw9GgwYNAABr165FaGgolEolfvjhB/zyyy9wc3PDc889x7JC9CQEIrIoERERAoAqN5lMVuHYwMBAYfz48YbXBw4cEAAIHTt2FPR6vWH/7du3BVtbW2HSpEmCIAiCTqcTfH19hTZt2gg6nc5wXE5OjuDp6Sl069bNsG/cuHFCo0aNDK/79+8vvPHGG4Krq6vwww8/CIIgCEePHhUACLt3737o93XixAkBgDBz5sxq/XuIj48XAAgRERGVPgZAmDNnjuH1nDlzBADCxx9/XOnY4cOHC/7+/hW+zz///FMAIPzxxx+CIAhCXl6e4ObmJgwePLjC5+p0OqFdu3bCU089Va3MRFSZxayoHDp0CIMHD4avry8kEgm2bt1q1OeXL/3+fXN0dKybwER17Mcff0R0dHSF7eTJk9X63JdffrnCqZXAwEB069YNBw4cAABcu3YNycnJeOWVVyCV/vWfEScnJ4wYMQInTpxAfn4+AKBfv364desW4uPjUVhYiCNHjmDgwIHo06cP9uzZA6B0lUUul6NHjx619e3XyIgRIyrtmzhxIu7evVthVSgiIgLe3t4ICwsDABw7dgyZmZkYP358hRUsvV6PgQMHIjo6mncjEdWQxVxMm5eXh3bt2mHixIlV/sfmcWbMmIE333yzwr5+/fqhS5cutRWRqF61aNGixhfTent7V7nv3LlzAID79+8DAHx8fCod5+vrC71ej6ysLDg4OBhO5+zduxfBwcEoLi5G3759ce/ePcyfP9/wse7du8Pe3v6hmRo2bAgAiI+Pr9H3VB1VfT9hYWHw8fFBREQEQkNDkZWVhd9//x1Tp06FTCYDANy7dw8AMHLkyIe+d2ZmJv/iQ1QDFlNUwsLCDH+7qUpRURFmz56NdevWITs7G61bt8aiRYvQu3dvAKV/E3RycjIcf+7cOVy+fBnffPNNXUcnMjmpqalV7nN3dwcAwz9TUlIqHZecnAypVApXV1cAgL+/P0JCQrB3714EBQWhc+fOcHFxQb9+/fD222/j5MmTOHHiBD755JNHZvLx8UGbNm2we/fuat2Ro1AoAABarbbC/vKSVZWqLtCVyWR45ZVX8OWXXyI7Oxvr16+HVqvFxIkTDceUX6eyfPlyPPPMM1W+t5eX1yPzElHVLObUz+NMnDgRR48excaNG3H+/HmMGjUKAwcORFxcXJXHr169GiEhIXj22WfrOSmR+DZs2ABBEAyv79y5g2PHjhmKfbNmzeDn54f169dXOC4vLw9btmwx3AlUrn///ti/fz/27NmDAQMGAABCQkLQsGFDfPzxxyguLn7khbTlPvroI2RlZeG9996r8HXL5ebmYvfu3QBKi4FCocD58+crHPPbb79V/19EmYkTJ6KwsBAbNmzAmjVr0LVr1wozarp37w4XFxdcvnwZnTt3rnKzs7Mz+usSkQWtqDzKzZs3sWHDBty9exe+vr4ASk/17Ny5ExEREVi4cGGF47VaLdatW4eZM2eKEZeoVly8eBElJSWV9jdu3BgeHh6P/Ny0tDQMGzYMb7zxBtRqNebMmQOFQoFZs2YBAKRSKRYvXoyxY8fihRdewOTJk6HVarFkyRJkZ2fjP//5T4X369evH77++mtkZGRg2bJlFfZHRETA1dX1sbcmA8CoUaPw0UcfYf78+bh69Spef/11w8C3kydP4n//+x9efPFFhIaGQiKRYNy4cfj+++/RuHFjtGvXDqdOncL69eur8W+voubNm6Nr164IDw9HYmIiVq1aVeHjTk5OWL58OcaPH4/MzEyMHDkSnp6eSE9Px7lz55Ceno6VK1ca/XWJCJZ51w8AITIy0vD6l19+EQAIjo6OFTYbGxth9OjRlT5//fr1go2NjZCSklKPqYlqx6Pu+gEgfPvtt4ZjH3bXz08//SS89957goeHhyCXy4Vnn31WOH36dKWvtXXrVuHpp58WFAqF4OjoKPTr1084evRopeOysrIEqVQqODo6CkVFRYb969atEwAIw4cPN+p7jIqKEkaOHCn4+PgItra2glKpFLp27SosWbJE0Gg0huPUarUwadIkwcvLS3B0dBQGDx4s3L59+6F3/aSnpz/0a65atUoAINjb2wtqtfqhuQYNGiS4ubkJtra2gp+fnzBo0CBh06ZNRn1/RPQXiSBUsX5q5iQSCSIjIzF06FAAwM8//4yxY8fi0qVLhovfyjk5OVW6cLB8YFNkZGR9RSYiIqIqWMWpnw4dOkCn0yEtLe2x15zEx8fjwIED+P333+spHRERET2MxRSV3NzcCmOu4+PjERsbCzc3N4SEhGDs2LF49dVXsXTpUnTo0AEZGRnYv38/2rRpg+eff97wed9//z18fHweeQcRERER1Q+LOfVz8OBB9OnTp9L+8ePHY82aNSguLsann36KH3/8EUlJSXB3d0fXrl3xySefoE2bNgAAvV6PwMBAvPrqq1iwYEF9fwtERET0NxZTVIiIiMjyWM0cFSIiIjI/LCpERERkssz6Ylq9Xo/k5GQ4OztXOfqaiIiITI8gCMjJyYGvr2+FB5tWxayLSnJyMgICAsSOQURERDWQmJgIf3//Rx5j1kXF2dkZQOk3qlQqRU5DRERE1aHRaBAQEGD4Pf4oZl1Uyk/3KJVKFhUiIiIzU53LNngxLREREZksFhUiIiIyWSwqREREZLJYVIiIiMhksagQERGRyWJRISIiIpPFokJEREQmi0WFiIiITBaLChEREZksFhUiIiIyWSwqREREZLJELSo5OTmYNm0aAgMDYW9vj27duiE6OlrMSERERGRCRC0qkyZNwp49e/DTTz/hwoULCA0NRf/+/ZGUlCRmLABAYmY+bqXnih2DiIjIqolWVAoKCrBlyxYsXrwYPXv2RJMmTTB37lwEBwdj5cqVYsUCAHx/JB7PLj6AL/bGiZqDiIjI2tmI9YVLSkqg0+mgUCgq7Le3t8eRI0eq/BytVgutVmt4rdFo6iRb5yBXAMDuS6nIKSyGs8K2Tr4OERERPZpoKyrOzs7o2rUr5s+fj+TkZOh0OqxduxYnT55ESkpKlZ8THh4OlUpl2AICAuokWxs/FRp7OEJboseOi6l18jWIiIjo8US9RuWnn36CIAjw8/ODXC7Hl19+iZdffhkymazK42fNmgW1Wm3YEhMT6ySXRCLB8I7+AIDIGPGvlyEiIrJWohaVxo0bIyoqCrm5uUhMTMSpU6dQXFyM4ODgKo+Xy+VQKpUVtroypL0vAOBE/H0kZxfU2dchIiKihzOJOSqOjo7w8fFBVlYWdu3ahSFDhogdCf6uDng62A2CAGyN5aoKERGRGEQtKrt27cLOnTsRHx+PPXv2oE+fPmjWrBkmTpwoZiyD4R39AJSe/hEEQeQ0RERE1kfUoqJWqzFlyhQ0b94cr776Knr06IHdu3fD1tY07rIJa+MDuY0UcWm5uJRcN3cYERER0cOJdnsyAIwePRqjR48WM8IjKRW26N/SC9vPp+DXmCS09lOJHYmIiMiqmMQ1KqZseIfS0z+/n0tGiU4vchoiIiLrwqLyGD1DPODmaIeMXC0O38gQOw4REZFVYVF5DFuZFIPb+gDgTBUiIqL6xqJSDcPKhr/tvpyKXG2JyGmIiIisB4tKNbTzV6FRA0cUFuuxkyP1iYiI6g2LSjVIJBIMK7uoNvLsXZHTEBERWQ8WlWoaWlZUjt28jxQ1R+oTERHVBxaVagpwc8BTQaUj9X+LTRY7DhERkVVgUTHCMI7UJyIiqlcsKkZ4vo0P7GykuHYvB5dTOFKfiIiorrGoGEFlb4v+LTwBcKYKERFRfWBRMdKwDqUzVX7jSH0iIqI6x6JipF4hHnB1sEV6jhZHb94XOw4REZFFY1Exkp2NFIPb+QIAImM4U4WIiKgusajUQPnwt12X7iGPI/WJiIjqDItKDbQPcEFwA0cUFOs4Up+IiKgOsajUQMWR+rz7h4iIqK6wqNRQeVE5ejMDqepCkdMQERFZJhaVGgpwc0CXINeykfpcVSEiIqoLLCpPoHymCk//EBER1Q0WlScwqI0P7GRSXE3NweVkjtQnIiKqbSwqT0DlYIt+5SP1z3KmChERUW1jUXlC5RfV/habDJ2eT1QmIiKqTSwqT6h3M0+4OtgiLUeLozcyxI5DRERkUVhUnpCdjRQvtC0bqc+LaomIiGoVi0otGNax9PTPzoupHKlPRERUi1hUakGHABcEuTugoFiHXZc4Up+IiKi2sKjUAolEgqEcqU9ERFTrWFRqiWGk/o0MpGk4Up+IiKg2sKjUkkB3R3QKdIVeKL1VmYiIiJ4ci0otKl9V+ZWnf4iIiGqFqEWlpKQEs2fPRnBwMOzt7dGoUSPMmzcPer1ezFg19kLb0pH6V1I0uJrKkfpERERPStSismjRInzzzTdYsWIFrly5gsWLF2PJkiVYvny5mLFqzMXBDn2aewAAImO4qkJERPSkRC0qx48fx5AhQzBo0CAEBQVh5MiRCA0NxenTp8WM9UTKn6i8NTaJI/WJiIiekKhFpUePHti3bx+uX78OADh37hyOHDmC559/vsrjtVotNBpNhc3U9GnuAZW9Le5ptDh+877YcYiIiMyaqEXlww8/xJgxY9C8eXPY2tqiQ4cOmDZtGsaMGVPl8eHh4VCpVIYtICCgnhM/ntxGhhfa+gAAfuUTlYmIiJ6IqEXl559/xtq1a7F+/XrExMTghx9+wGeffYYffvihyuNnzZoFtVpt2BITE+s5cfUMf2Ckfn4RR+oTERHVlI2YX/yDDz7AzJkz8dJLLwEA2rRpgzt37iA8PBzjx4+vdLxcLodcLq/vmEbr2NAVge4OuHM/H7sv3TNMrSUiIiLjiLqikp+fD6m0YgSZTGa2tyeXk0gkGNqeM1WIiIielKhFZfDgwViwYAG2b9+O27dvIzIyEp9//jmGDRsmZqxaUT787UhcOkfqExER1ZCoRWX58uUYOXIk3n77bbRo0QIzZszA5MmTMX/+fDFj1YqgBo7o2NAFegH4/RxH6hMREdWERBAEsx32odFooFKpoFaroVQqxY5TyU8n7uCjrRfR0keJP6c+K3YcIiIik2DM728+66cOvdDGB7YyCS6naHAtNUfsOERERGaHRaUOuTraoU8zTwCcqUJERFQTLCp1rHymym9nkzlSn4iIyEgsKnWsT3NPqOxtkaopxIlbHKlPRERkDBaVOia3kWFQ+Uh9PlGZiIjIKCwq9WB4h/KR+ikoKNKJnIaIiMh8sKjUg06Brmjo5oC8Ih12X04VOw4REZHZYFGpBxKJxPC8H57+ISIiqj4WlXpSPlL/cFw60nI4Up+IiKg6WFTqSXADR7QPKBupH8uR+kRERNXBolKPymeqbI3l6R8iIqLqYFGpRy+09YWNVIKLSRrE3eNIfSIiosdhUalHbo526G0Yqc9VFSIiosdhUalnf43UT4KeI/WJiIgeiUWlnvVt7glnhQ2S1YU4Ec+R+kRERI/ColLPFLYyvFA2Uj+SM1WIiIgeiUVFBMM6+AMAdlxM5Uh9IiKiR2BREUHnQFf4u9ojV1uCPVfuiR2HiIjIZLGoiEAqlRgm1UbG3BU5DRERkeliURFJeVE5FJeB9BytyGmIiIhME4uKSBp5OKFdgAt0egF/nONIfSIioqqwqIhoePnpHw5/IyIiqhKLiogGtysdqX8hSY0baRypT0RE9HcsKiIqHanvAQD4lTNViIiIKmFREVn5TJXfYpM5Up+IiOhvWFRE1q9F6Uj9pOwCnIzPFDsOERGRSWFREZnCVoZBbcpG6p/lTBUiIqIHsaiYgPKZKjsupKKwmCP1iYiIyrGomIAuQW7wc7FHjrYEey5zpD4REVE5FhUTUGGkPmeqEBERGbComIhhHUuLStT1dGTkcqQ+ERERIHJRCQoKgkQiqbRNmTJFzFiiaOzhhHb+Ko7UJyIieoCoRSU6OhopKSmGbc+ePQCAUaNGiRlLNDz9Q0REVJGoRcXDwwPe3t6Gbdu2bWjcuDF69eolZizRvNDOFzKpBOfvqnEjLVfsOERERKIzmWtUioqKsHbtWrz22muQSCRVHqPVaqHRaCpslqSBkxy9QkpH6m/lqgoREZHpFJWtW7ciOzsbEyZMeOgx4eHhUKlUhi0gIKD+AtaTB0//cKQ+ERFZO5MpKt999x3CwsLg6+v70GNmzZoFtVpt2BITE+sxYf0Y0NILzvLSkfrRtzlSn4iIrJtJFJU7d+5g7969mDRp0iOPk8vlUCqVFTZLo7CVIayNNwBeVEtERGQSRSUiIgKenp4YNGiQ2FFMQvkTlbdfSOFIfSIismqiFxW9Xo+IiAiMHz8eNjY2YscxCU8Hl43ULyzBvitpYschIiISjehFZe/evUhISMBrr70mdhSTIZVKMKR96bU6fKIyERFZM9GLSmhoKARBQEhIiNhRTMrwspH6B6+l4z5H6hMRkZUSvahQ1Zp4OqONnwolegHbzqeIHYeIiEgULComrHymyq+8+4eIiKwUi4oJ+0f70pH65xKzcTOdI/WJiMj6sKiYsAZOcvRs2gAAR+oTEZF1YlExccM6ls5U4Uh9IiKyRiwqJi60pRec5Da4m1WA03eyxI5DRERUr1hUTJzCVoaw1uUj9TlThYiIrAuLihkYVjZTZdt5jtQnIiLrwqJiBp4JdoevSoGcwhLsv8qR+kREZD1YVMyAVCrBkPKZKjG8+4eIiKwHi4qZGN6hfKR+GjLzikROQ0REVD9YVMxEUy9ntPZTlo3UTxY7DhERUb1gUTEjwzqUzlTh6R8iIrIWLCpm5B/tSkfqxyZm4xZH6hMRkRVgUTEjHs5yPMuR+kREZEVYVMxM+ROVI2OTIAgcqU9ERJaNRcXMhLb0hqOdDImZHKlPRESWj0XFzNjbyTCwtQ+A0gcVEhERWTIWFTM0vGyk/vbzKdCWcKQ+ERFZLhYVM/RMI3d4KxVQFxTjAEfqExGRBWNRMUMyqQRDOvgC4EwVIiKybCwqZmp42fC3A9fSkMWR+kREZKFYVMxUM29ntPRRolgnYNuFFLHjEBER1QkWFTNWflFtZMxdkZMQERHVDRYVM/aPdr6QSoCYhGzczsgTOw4REVGtY1ExY55KBXo09QDAmSpERGSZWFTM3PCykfpbOVKfiIgsEIuKmQtt5QUHOxnu3M9HTAJH6hMRkWVhUTFzDnY2GNjaGwBnqhARkeVhUbEA5TNVtnGkPhERWRgWFQvQtbE7vJTyspH66WLHISIiqjWiF5WkpCSMGzcO7u7ucHBwQPv27XHmzBmxY5kVmVSCoe3LZqqc5UwVIiKyHKIWlaysLHTv3h22trbYsWMHLl++jKVLl8LFxUXMWGZpWNnwt/1X05Cdz5H6RERkGWxq8knZ2dnYvHkzbt68iQ8++ABubm6IiYmBl5cX/Pz8qv0+ixYtQkBAACIiIgz7goKCahLJ6jX3VqKFjxJXUjTYdj4F454JFDsSERHREzN6ReX8+fMICQnBokWL8NlnnyE7OxsAEBkZiVmzZhn1Xr///js6d+6MUaNGwdPTEx06dMC3335rbCQqUz5ThcPfiIjIUhhdVKZPn44JEyYgLi4OCoXCsD8sLAyHDh0y6r1u3bqFlStXomnTpti1axfefPNNvPfee/jxxx+rPF6r1UKj0VTY6C9D2peO1D9zJwt37nOkPhERmT+ji0p0dDQmT55cab+fnx9SU1ONei+9Xo+OHTti4cKF6NChAyZPnow33ngDK1eurPL48PBwqFQqwxYQEGBsfIvmqVSge5MGALiqQkRElsHooqJQKKpcybh27Ro8PDyMei8fHx+0bNmywr4WLVogISGhyuNnzZoFtVpt2BITE436etbA8ETlsxypT0RE5s/oojJkyBDMmzcPxcXFAACJRIKEhATMnDkTI0aMMOq9unfvjmvXrlXYd/36dQQGVn0hqFwuh1KprLBRRc+18n5gpH622HGIiIieiNFF5bPPPkN6ejo8PT1RUFCAXr16oUmTJnB2dsaCBQuMeq/3338fJ06cwMKFC3Hjxg2sX78eq1atwpQpU4yNRWUc7GwwsFXpSH3OVCEiInMnEWp4fmD//v2IiYkxXGfSv3//GgXYtm0bZs2ahbi4OAQHB2P69Ol44403qvW5Go0GKpUKarWaqysPOByXjle+OwUXB1uc+ld/2NmIPtePiIjIwJjf30YVlZKSEigUCsTGxqJ169ZPHPRJsahUTacX0DV8H9JytFj1SieElq2wEBERmQJjfn8b9VdtGxsbBAYGQqfjg+9MmUwqwZD2vgB49w8REZk3o88JzJ49G7NmzUJmZmZd5KFaMqzsicr7rqRBnV8schoiIqKaMXqE/pdffokbN27A19cXgYGBcHR0rPDxmJiYWgtHNdfSV4nm3s64mpqD7RdS8PLTDcWOREREZDSji8rQoUPrIAbVhWEd/BC+4yo2RidgzFMBkEgkYkciIiIySo3v+jEFvJj20dJztOi15ADyi3T470vtMaR99R8YSUREVFfq7GLaB505cwZr167FunXrcPbs2Zq+DdUhD2c5pvRpAgD4z46ryC8qETkRERGRcYw+9ZOWloaXXnoJBw8ehIuLCwRBgFqtRp8+fbBx40ajx+hT3Xq9RzA2nErA3awCfBN1C9MHhIgdiYiIqNqMXlF59913odFocOnSJWRmZiIrKwsXL16ERqPBe++9VxcZ6QkobGWYPagFAOB/UTdxNytf5ERERETVZ3RR2blzJ1auXIkWLVoY9rVs2RJfffUVduzYUavhqHY818obzzRyg7ZEj/AdV8WOQ0REVG1GFxW9Xg9bW9tK+21tbaHX62slFNUuiUSCj19oBakE2H4+BafiOQOHiIjMg9FFpW/fvpg6dSqSk5MN+5KSkvD++++jX79+tRqOak9LXyXGPFU6S+WTPy5Bpzfbm72IiMiKGF1UVqxYgZycHAQFBaFx48Zo0qQJgoODkZOTg+XLl9dFRqol0weEQKmwwaVkDTadThQ7DhER0WPVeI7Knj17cPXqVQiCgJYtW9b46clPgnNUjPf9kXjM23YZ7o52OPBBbygVlU/jERER1aU6e3qyqWFRMV6xTo+Byw7hZnoe3ng2GP8e1FLsSEREZGXqdODbe++9hy+//LLS/hUrVmDatGnGvh3VM1uZFB+9UFpOIo7exq30XJETERERPZzRRWXLli3o3r17pf3dunXD5s2bayUU1a3ezTzRt7knSvQCFmy/InYcIiKihzK6qNy/fx8qlarSfqVSiYyMjFoJRXVv9qAWsJFKsO9qGg5eSxM7DhERUZWMLipNmjTBzp07K+3fsWMHGjVqVCuhqO418nDCxO5BAID52y6jWMcZOEREZHqMftbP9OnT8c477yA9PR19+/YFAOzbtw9Lly7FsmXLajsf1aF3+zXFrzFJuJmeh5+O38FrPYLFjkRERFRBje76WblyJRYsWGAY+hYUFIS5c+fi1VdfrfWAj8K7fp7chlMJmPXrBSgVNjj4QR+4OdqJHYmIiCxcvd2enJ6eDnt7ezg5OdX0LZ4Ii8qT0+kFDF5+BJdTNBj3TEN8OrSN2JGIiMjC1entyQ/y8PDAmTNnsGPHDmRlZT3JW5FIZFIJ5gwuvV15/ckEXEnRiJyIiIjoL9UuKkuWLMGcOXMMrwVBwMCBA9GnTx8MGjQILVq0wKVLl+okJNWtpxu5Y1BbH+gFYN4fl2HGMwCJiMjCVLuobNiwAS1b/jXFdPPmzTh06BAOHz6MjIwMdO7cGZ988kmdhKS6NyusOeQ2Uhy/dR+7LqWKHYeIiAiAEUUlPj4ebdu2Nbz+888/MWLECHTv3h1ubm6YPXs2jh8/Xichqe75uzpgcs/S28s/3X4FhcU6kRMREREZUVSKi4shl8sNr48fP45u3boZXvv6+nLgm5l7s3djeCsVuJtVgO+OxIsdh4iIqPpFpUmTJjh06BAAICEhAdevX0evXr0MH7979y7c3d1rPyHVGwc7G8x6vjkA4KsDN3BPUyhyIiIisnbVLipvvfUW3nnnHbz++usICwtD165dK1yzsn//fnTo0KFOQlL9+Uc7X3QKdEV+kQ6Ldl4VOw4REVm5aheVyZMn47///S8yMzPRs2dPbNmypcLHk5OT8dprr9V6QKpfEslftyv/GpOEswm87ZyIiMTzRAPfxMaBb3VnxqZz2HzmLtoHuODXt7pBKpWIHYmIiCxEvQ18I8v1z+eawdFOhtjEbGyNTRI7DhERWSlRi8rcuXMhkUgqbN7e3mJGojKeSgXe6dsUALBo51XkaUtETkRERNZI9BWVVq1aISUlxbBduHBB7EhU5rUeQQh0d8A9jRYrD94UOw4REVkh0YuKjY0NvL29DZuHh4fYkaiM3EaGfz/fAgCw6vAtJGbmi5yIiIisjehFJS4uDr6+vggODsZLL72EW7duiR2JHjCgpRe6N3FHUYkeC/+8InYcIiKyMjW66yc6OhqbNm1CQkICioqKKnzs119/rfb77NixA/n5+QgJCcG9e/fw6aef4urVq7h06VKVw+O0Wi20Wq3htUajQUBAAO/6qWPXUnMQ9t9D0AvA+jeeRrfGDcSOREREZqxO7/rZuHEjunfvjsuXLyMyMhLFxcW4fPky9u/fD5VKZdR7hYWFYcSIEWjTpg369++P7du3AwB++OGHKo8PDw+HSqUybAEBAcbGpxpo5u2Mcc8EAih9urJOb7Z3tBMRkZkxuqgsXLgQX3zxBbZt2wY7Ozv897//xZUrVzB69Gg0bNjwicI4OjqiTZs2iIuLq/Ljs2bNglqtNmyJiYlP9PWo+t7vHwKVvS2upuZgY3SC2HGIiMhKGF1Ubt68iUGDBgEA5HI58vLyIJFI8P7772PVqlVPFEar1eLKlSvw8fGp8uNyuRxKpbLCRvXD1dEO0weEAAA+23UN6vxikRMREZE1MLqouLm5IScnBwDg5+eHixcvAgCys7ORn2/cXSEzZsxAVFQU4uPjcfLkSYwcORIajQbjx483NhbVg7FPN0RTTydk5Rfjv/uqXvUiIiKqTUYXlWeffRZ79uwBAIwePRpTp07FG2+8gTFjxqBfv35Gvdfdu3cxZswYNGvWDMOHD4ednR1OnDiBwMBAY2NRPbCRSfFx2XOAfjx+GzfSckROREREls7ou34yMzNRWFgIX19f6PV6fPbZZzhy5AiaNGmCjz76CK6urnWVtRI+60cck344jb1X7qFXiAd+eO0pseMQEZGZMeb3Nx9KSEa7nZGHAV9EoVgnIGJCF/Rp7il2JCIiMiO1fnuyRqOp8OdHbWT5gho44rUewQCA+dsuo6hEL3IiIiKyVNUqKq6urkhLSwMAuLi4wNXVtdJWvp+swzt9mqCBkxy3MvLw4/HbYschIiILZVOdg/bv3w83NzcAwIEDB+o0EJkHZ4Ut/vlcM/xzy3n8d28chnbwQwMnudixiIjIwlSrqPTq1cvw5+DgYAQEBEAikVQ4RhAEDmCzMiM7+eOnE3dwIUmNpbuvIXx4W7EjERGRhTH69uTg4GCkp6dX2p+ZmYng4OBaCUXmQSqVYE7Z7coboxNxKVktciIiIrI0RhcVQRAqraYAQG5uLhQKRa2EIvPROcgN/2jnC0EAPvnjMsz4JjIiIjJB1Tr1AwDTp08HAEgkEnz00UdwcHAwfEyn0+HkyZNo3759rQck0zczrDl2X07FqfhM/HkhFYPaVv0IBCIiImNVu6icPXsWQOmKyoULF2BnZ2f4mJ2dHdq1a4cZM2bUfkIyeb4u9nizV2Ms2xuHhX9eQb8WnlDYysSORUREFqDaRaX8bp8JEyZg+fLlcHZ2rrNQZH4m92yMX6ITkZRdgFWHbuG9fk3FjkRERBbAqGtUSkpKsHbtWty5c6eu8pCZsreTYdbzLQAAKw/eRIq6QORERERkCYwqKjY2NggMDIROp6urPGTGXmjrgy5Brigo1mHRjqtixyEiIgtg9F0/s2fPxqxZs5CZmVkXeciMSSQSzBncChIJsDU2GWfu8H8jRET0ZIx+KGGHDh1w48YNFBcXIzAwEI6OjhU+HhMTU6sBH4UPJTRNH24+j59PJ6Ktvwpb3+4OqbTy7exERGS9jPn9Xe2LacsNHTq0prnISsx4rhm2X0jB+btqbIm5i1GdA8SOREREZsroFRVTwhUV07Xq0E0s/PMqPJzlODCjN5zkRndiIiKyUMb8/jb6GhUAyM7OxurVqytcqxITE4OkpKSavB1ZoAndghHcwBHpOVp8deCG2HGIiMhMGV1Uzp8/j5CQECxatAifffYZsrOzAQCRkZGYNWtWbecjM2VnI8XsQaW3K393OB537ueJnIiIiMyR0UVl+vTpmDBhAuLi4io82ycsLAyHDh2q1XBk3vo298SzTRugSKfHgu1XxI5DRERmyOiiEh0djcmTJ1fa7+fnh9TU1FoJRZZBIpHg4xdaQiaVYPflezgSlyF2JCIiMjNGFxWFQgGNRlNp/7Vr1+Dh4VErochyNPVyxivPBAIA5m27hBKdXuRERERkTowuKkOGDMG8efNQXFwMoPRvzQkJCZg5cyZGjBhR6wHJ/L3fPwSuDra4fi8X608liB2HiIjMiNFF5bPPPkN6ejo8PT1RUFCAXr16oUmTJnB2dsaCBQvqIiOZOZWDLaaHNgMAfL7nOrLzi0RORERE5sLo4RZKpRJHjhzB/v37ERMTA71ej44dO6J///51kY8sxJguAVh34g6upuZg2d44zP1HK7EjERGRGeDAN6o3x25k4OXVJyGTSrBj6rMI8XIWOxIREYmgTkfoA8C+ffuwb98+pKWlQa+veHHk999/X5O3JCvQrUkDPNfKC7su3cP8bZfx42tPQSLhc4CIiOjhjL5G5ZNPPkFoaCj27duHjIwMZGVlVdiIHuXfz7eEnUyKw3EZ2HslTew4RERk4oxeUfnmm2+wZs0avPLKK3WRhyxcQ3cHTHo2GF8fvIkF2y+jZ0gDyG1kYsciIiITZfSKSlFREbp161YXWchKvN2nCTyd5bh9Px9rjt4WOw4REZkwo4vKpEmTsH79+rrIQlbCSW6Dfw5sDgBYvv8G0nIKRU5ERESmyuhTP4WFhVi1ahX27t2Ltm3bwtbWtsLHP//881oLR5ZreAc//HT8Ns7dVeOzXdeweGQ7sSMREZEJqtHTk9u3bw+pVIqLFy/i7Nmzhi02NrbGQcLDwyGRSDBt2rQavweZD6lUgo8Hl85S2XTmLs7fzRY3EBERmSSjV1QOHDhQ6yGio6OxatUqtG3bttbfm0xXp0BXDOvgh8izSZj3x2VserMrb1cmIqIKjF5RqW25ubkYO3Ysvv32W7i6uoodh+rZhwObw95WhtN3svDH+RSx4xARkYmp9orK8OHDq3Xcr7/+alSAKVOmYNCgQejfvz8+/fRToz6XzJ+3SoG3ezfG0j3XEf7nFQxo4QV7O96uTEREpapdVFQqVa1/8Y0bNyImJgbR0dHVOl6r1UKr1RpeazSaWs9E9e+Nno2wMToRSdkF+CbqJt4fECJ2JCIiMhHVLioRERG1+oUTExMxdepU7N69GwqFolqfEx4ejk8++aRWc5D4FLYy/HtQC7y9LgbfRN3E6C4B8HOxFzsWERGZANEeSrh161YMGzYMMtlfy/w6nQ4SiQRSqRRarbbCx4CqV1QCAgL4UEILIAgCXlp1AifjM/FCWx+seLmj2JGIiKiOGPNQQtEupu3Xrx8uXLiA2NhYw9a5c2eMHTsWsbGxlUoKAMjlciiVygobWQaJRIKPB7eEVAJsO5+CU/GZYkciIiITIFpRcXZ2RuvWrStsjo6OcHd3R+vWrcWKRSJq5avCi10aAgA++eMSdHpRFvuIiMiEiH57MtGDZoSGwFlhg0vJGmw+kyh2HCIiEplJFZWDBw9i2bJlYscgEbk7yTG1X1MAwJJd16ApLBY5ERERicmkigoRALzaNQiNPByRkVuEFftviB2HiIhExKJCJsfORoqPXmgJAIg4Go/4jDyRExERkVhYVMgk9WnmiT7NPFCsE7Bg+2Wx4xARkUhYVMhkzX6hJWykEuy9koao6+lixyEiIhGwqJDJauzhhPHdggAA87ddRrFOL24gIiKqdywqZNLe69cUbo52uJGWi5+O3xE7DhER1TMWFTJpKntbzAhtBgD4z86r+ONcssiJiIioPrGokMl7sUsAQlt6oahEj3c3nMVXB25ApEdUERFRPWNRIZMnk0qwclwnTOweBKB0ENzMLRd4zQoRkRVgUSGzIJNKMGdwK8wte3Dhz6cTMSHiFNQFnFxLRGTJWFTIrEzoHoxvX+0MBzsZjt64j5Erj+FuVr7YsYiIqI6wqJDZ6dfCC79M7gpPZzni0nIx9KtjOJeYLXYsIiKqAywqZJZa+6mwdUp3NPd2RkauFi+uOo6dF1PFjkVERLWMRYXMlq+LPTa/1Q29QjxQWKzHW+vOYPXhW7wjiIjIgrCokFlzktvgu/GdMfbphhAE4NPtV/Dxb5dQwjuCiIgsAosKmT0bmRSfDm2Nfz/fAhIJ8NOJO5j042nkakvEjkZERE+IRYUsgkQiwRs9G2Hl2I5Q2Epx8Fo6Rn1zHCnqArGjERHRE2BRIYsysLUPNv5fVzRwssOVFA2GfnUUF5PUYsciIqIaYlEhi9M+wAWRb3dHU08n3NNoMfp/x7H/6j2xYxERUQ2wqJBFCnBzwOa3uqF7E3fkF+kw6YfT+PH4bbFjERGRkVhUyGKp7G0RMeEpjOrkD70AfPzbJczfdhk6PW9fJiIyFywqZNHsbKRYPLItPniuGQDguyPxeHPtGeQX8Y4gIiJzwKJCFk8ikWBKnyb4ckwH2NlIsefyPby06gTScgrFjkZERI/BokJW4x/tfLF+0tNwdbDF+btqDPvqGK6l5ogdi4iIHoFFhaxK5yA3RL7dHcENHJGUXYCRK4/h0PV0sWMREdFDsKiQ1Qlq4Ihf3+qGp4LckKMtwcQ10dh4KkHsWEREVAUWFbJKro52+GnSUxja3hc6vYCZv17Aop1XoecdQUREJoVFhayW3EaGL15sj6n9mgIAVh68iXc3nEVhsU7kZEREVI5FhayaRCLB+wNCsHRUO9jKJNh+IQUvf3sC93O1YkcjIiKwqBABAEZ08sePrz0NpcIGMQnZGPb1MdxIyxU7FhGR1WNRISrTtbE7fn27OwLc7JGQmY/hXx/F8Zv3xY5FRGTVWFSIHtDE0wlb3+6ODg1doCkswavfn8SWM3fFjkVEZLVELSorV65E27ZtoVQqoVQq0bVrV+zYsUPMSERwd5JjwxvPYFAbHxTrBPy/Tefw+Z7rEATeEUREVN9ELSr+/v74z3/+g9OnT+P06dPo27cvhgwZgkuXLokZiwgKWxmWj+mAt3o3BgB8uS8O7/8cC20J7wgiIqpPEsHE/pro5uaGJUuW4PXXX3/ssRqNBiqVCmq1Gkqlsh7SkTXacCoBs7dehE4v4KlgN6x6pRNcHOzEjkVEZLaM+f1tMteo6HQ6bNy4EXl5eejatWuVx2i1Wmg0mgobUV0b81RDrJnYBc5yG5yKz8Twr4/hdkae2LGIiKyC6EXlwoULcHJyglwux5tvvonIyEi0bNmyymPDw8OhUqkMW0BAQD2nJWv1bFMPbH6rG/xc7HErIw/Dvj6K07czxY5FRGTxRD/1U1RUhISEBGRnZ2PLli1YvXo1oqKiqiwrWq0WWu1fg7g0Gg0CAgJ46ofqTZqmEK//cBoXktSws5Fi6ah2GNzOV+xYRERmxZhTP6IXlb/r378/GjdujP/973+PPZbXqJAY8otKMHVjLPZcvgcA+OC5Zni7d2NIJBKRkxERmQezvEalnCAIFVZNiEyNg50NvhnXCa/3CAYALNl1DR9uOY9inV7kZERElsdGzC/+r3/9C2FhYQgICEBOTg42btyIgwcPYufOnWLGInosmVSCj15oiUB3B8z9/RJ+OX0XSdkF+HpsJ6jsbcWOR0RkMURdUbl37x5eeeUVNGvWDP369cPJkyexc+dODBgwQMxYRNX2atcgrB7fGQ52Mhy9cR8jVx5DYma+2LGIiCyGyV2jYgxeo0Km4mKSGq//EI17Gi0aONlh9fguaB/gInYsIiKTZNbXqBCZo9Z+Kmyd0h0tfJTIyC3CS6uOY+fFVLFjERGZPRYVolrio7LHpje7onczDxQW6/HWujP49tAtPiOIiOgJsKgQ1SInuQ1Wv9oZ455pCEEAFvx5BbO3XkQJ7wgiIqoRFhWiWmYjk2L+kNaYPagFJBJg3ckETPrxNHK1JWJHIyIyOywqRHVAIpFg0rONsHJsJyhspTh4LR0jVx5DirpA7GhERGaFRYWoDg1s7Y2f/68rGjjJcTU1B0O/OoqLSWqxYxERmQ0WFaI61i7ABZFvd0NTTyfc02gx7OujmPv7JWTkcgIzEdHjsKgQ1YMANwdsfqsb+rfwQrFOwJpjt9Fr8QH8d28c8njtChHRQ3HgG1E9O3YjA+E7ruJC2SmgBk5yTO3fFC91CYCtjH93ICLLZ9ZPTzYGiwqZK71ewPYLKViy6xoSykbuBzdwxAfPNUNYa28+iZmILBqLCpGZKCrRY8OpBHy5Lw7384oAlF7TMiusOZ5p5C5yOiKiusGiQmRmcrUlWHXoFlYfvoX8Ih0AoG9zT/xzYDM09+b/tonIsrCoEJmptJxCfLkvDhtOJUKnFyCRACM6+uP9ASHwc7EXOx4RUa1gUSEyc7fSc7F093Vsv5ACALCzkWJCtyC83bsxXBzsRE5HRPRkWFSILERsYjbC/7yCk/GZAAClwgZv92mCCd2CoLCViZyOiKhmWFSILIggCDh4LR3/2XEV1+7lAAB8VAq8PyAEIzr6QyblHUJEZF5YVIgskE4vIPJsEj7ffQ3J6kIAQIiXEz4c2Bx9m3vylmYiMhssKkQWrLBYhx+P38ZXB25CXVAMAHgq2A0zw5qjY0NXkdMRET0eiwqRFVDnF+PrqBuIOHobRSV6AEBYa2/MeK4ZGns4iZyOiOjhWFSIrEhydgGW7b2OzWfuQi8AMqkEL3YJwLR+TeGpVIgdj4ioEhYVIit0LTUHS3Zdxd4raQAAe1sZJj0bjP/r2QjOCluR0xER/YVFhciKnbx1H//ZeRVnE7IBAG6Odni3bxOMfToQdjZ86CERiY9FhcjKCYKAXZdSsXjnNdzKyAMABLjZY0ZoMwxu6wspb2kmIhGxqBARAKBYp8cvpxOxbG8c0nO0AIDWfkrMHNgCPZo2EDkdEVkrFhUiqiC/qATfHY7H/w7dQq62BADwbNMG+HBgc7T2U4mcjoisDYsKEVXpfq4Wy/ffwLqTd1CsK/2//pD2vpgR2gwBbg4ipyMia8GiQkSPlHA/H0v3XMNvsckAAFuZBOOeCcS7fZvCzZEPPSSiusWiQkTVcjFJjf/suIojNzIAAM5yG0zu1Qiv9QiGg52NyOmIyFKxqBCRUQ7HlT708FKyBgDg6SzHtP4hGN3ZHzYy3tJMRLWLRYWIjKbXC/jjfDKW7LqGu1kFAIBGHo7453PN8VwrLz70kIhqDYsKEdWYtkSHdScSsHx/HLLySx962LGhC2Y93wJdgtxETkdElsCY39+irumGh4ejS5cucHZ2hqenJ4YOHYpr166JGYnI6sltZHitRzCi/tkH7/RpAoWtFDEJ2Rj1zXFM+iEacfdyxI5IRFZE1KISFRWFKVOm4MSJE9izZw9KSkoQGhqKvLw8MWMREQClwhYznmuGqA/64OWnG0ImlWDvlTQ8t+wQ/rn5HJKyC8SOSERWwKRO/aSnp8PT0xNRUVHo2bPnY4/nqR+i+nMjLRdLdl3Frkv3DPsaeTjiqSA3dA5yw1NBbghws+e1LET0WMb8/jap+w/VajUAwM2t6vPgWq0WWq3W8Fqj0dRLLiICmng64X+vdMaZO1lYsusqTtzKxK30PNxKz8PG6EQApXcLdQl2Q5dAV3QJdkNzbyVkfK4QET0Bk1lREQQBQ4YMQVZWFg4fPlzlMXPnzsUnn3xSaT9XVIjqX3Z+EU7fzkL0nUxEx2fiQpLaMO22nJPcBh0DXfFUkCs6B7mhfYALFLYykRITkakwy7t+pkyZgu3bt+PIkSPw9/ev8piqVlQCAgJYVIhMQGGxDrGJ2Th9OxPRt7Nw5k6W4blC5WxlErTxU5Wturihc5ArXBw4CZfI2phdUXn33XexdetWHDp0CMHBwdX+PF6jQmS6dHoBV1M1iI7PRPSdLETHZyItR1vpuBAvJ3QJcivdgt3g52IvQloiqk9mU1QEQcC7776LyMhIHDx4EE2bNjXq81lUiMyHIAhIzCzAqduZOH07E6dul17j8ne+KgU6l5WWLkGuCPF0hpTXuRBZFLMpKm+//TbWr1+P3377Dc2aNTPsV6lUsLd//N+qWFSIzNv9XC1Ol622RN/JwsUkNXT6iv9JUipsSotLUGlxaeOvgtyG17kQmTOzKSoPu40xIiICEyZMeOzns6gQWZb8ohLEJmSXrbpkISYhC/lFugrH2NlI0d7fBV2CSy/Q7RToCqXCVqTERFQTZlNUnhSLCpFlK9bpcSVFg1PxpcUl+nYm7ucVVThGIgGaeyvRJcjVcK2Lt0ohUmIiqg4WFSKySIIgID4jD6dvZxmudbl9P7/ScQFu9ugS+Nd1Lo09nDiIjsiEsKgQkdVI0xTi9J2s0lWXO5m4nKzB3y5zgauDbdl1LqWrLq39VLCVifoEESKrxqJCRFYrp7AYZxOyEX07E9G3M3E2IRvaEn2FYxS2UnQIcEWXIFd0CnJDWz8VXB05z4WovrCoEBGVKSrR42KyuvSW6PgsnL6Tiez84krH+bvao42fCq39VGhTtrG8ENUNFhUioofQ6wXcTM9FdNnFuWcTsqq8zgVgeSGqKywqRERGUBcU41KyGheT1Dh/t/Sfjysvbfz/Ki98DACRcVhUiIieUHl5uXBXjQtJjy8vbf0rrrywvBA9HIsKEVEdUBcU41JSaXF5XHkJcKt82ojlhagUiwoRUT15sLycLysvd6pRXtr6uaC1n5LlhawSiwoRkYjKy8v5B1ZeHlde2vi5lJUYlheyfCwqREQmRp1fjIvJFU8bPaq8lK64/HXaSOXA5xmR5WBRISIyA38vLxfuqpGQWXV5aejmUOmaF5YXMlcsKkREZqpCeSm746g65aW1nxLNvJzh4Sznc43I5LGoEBFZkOz8IlxM0hhOGT2qvKjsbdHU0wlNvZzR1NMJIV7OaOrlBE8WGDIhLCpERBbu7+XlcooGd+7nVXogYzmlwsZQXh4sMV5KFhiqfywqRERWqLBYh1vpeYhLy0HcvVzDP28/osA4K2xKy4tn6cpLUy9nhHg5wVupYIGhOsOiQkREBtqS8gKTi7h7peXleloO7tzPh+4hDcZZboMmXk6GlZcmZf/0UbHA0JNjUSEiosfSluhwOyMf1+/l/FVi0nJxOyMPJQ8pME5yGzTxdCo7hfTXaSQ/F3sWGKo2FhUiIqqxohI9bt/PKy0wD5xCin9EgXG0k5UWmAeuf2lSVmCkUhYYqsiY39829ZSJiIjMhJ2NFCFezgjxcq6wv6hEjzv3S08hPbgKE5+Rh7wiHc7dVePcXXWFz3EoLzBl18CEeJX+mQWGqosrKkRE9ESKdWUF5l4urpetwNxIy8Wt9DwU6fRVfo69reyBU0h/rcL4u7LAWAOe+iEiItGV6PS4k5n/wAW8pSswjyowchspfF3s4a1UwMdFAV+VPXxcFPBRKeCjsoevyh5KexteD2PmWFSIiMhklej0SMjMx/V7ubiRllO2CpOLm+m5KCqpusA8yMFOBm9VWYlRKeDjUvZPlQK+ZX92VvDxAqaM16gQEZHJspFJ0cjDCY08nAB4G/br9AKSsgqQrC5AqroQyeoCpGQXIkVdgOTsQqRqCpGZV4T8otLbrW+l5z30azjLbeBdVmJ8y1ZjSktN2cqMiwIOdvwVaA74UyIiIpMgk0rQ0N0BDd0dHnpMYbEOKepCpGQXlP5TXYDkCq8LoS4oRo62BDlppSs1D6NU2BhWYHxc7OGjfKDYlO1X2Mrq4lslI7CoEBGR2VDYyhDcwBHBDRwfekyetsRQYkpLTcVCk6ouRI62BJrCEmhSc3A1Neeh7+XqYGtYgfFW/bUaU369jJdKDrkNy0xdYlEhIiKL4lg2lK6Jp9NDj8kpLEaKuhDJD6zElK/KlJ9yKijWISu/GFn5xbiconnoezVwsvvr1NIDqzHeSgW8lKUFhyszNceiQkREVsdZYQtnhW2lWTHlBEGApqCktLSUXyPzt+tmUtSF0JbokZFbhIzcIlxIUlf5XkDpU629lPLS4lJWYLwMZUYOb6UC7k5yyHhrdiUsKkRERH8jkUigcrCFysEWLXyqvitFEARk5Rc/sCpTYFiZSVYXIk1TegFwYbEe6oJiqAuKcf3ew6+ZkUkl8HCSlxWY0lLzYLHxVpXuc5Jb1+3ZLCpEREQ1IJFI4OZoBzdHO7T2U1V5TPnKzL2c0hWZVM1fBSZVrUVa2f6MXC10eqF0v6YQ5x7xdR3sZPBWKuBZthLjpVLAy7n0FJNX2QqNp7MCdjbSuvnG6xmLChERUR15cGXmYaeZgNLZMhm5RWUFptBQYO5ptLhXVl7uaQqRU1hSent2Rh5uZTz89myg9NoZz78VmL8XG1cHW5NfnRG1qBw6dAhLlizBmTNnkJKSgsjISAwdOlTMSERERPXORiaFt6q0PCDg4cflaUtwT1O5wNzT/FVs0nIKUawTDNfOPOpCYDuZtIqVmYqnncS+GFjUopKXl4d27dph4sSJGDFihJhRiIiITJ6j3OaBYXlV0+sFZOYXPVBgtIY/l5ab0teZeUUo0ulxN6sAd7MKHvp+/Vt4YvX4LnXx7VSLqEUlLCwMYWFhYkYgIiKyKFKpBA2c5GjgJEcr36qvnQEAbYkOaRqtYYWm8upM6YqNp1JRj+krM6trVLRaLbRareG1RvPw5SwiIiJ6OLmNDAFuDghwe/gkYEEQUKwT95GAZnVJcHh4OFQqlWELCHjEiTwiIiJ6IhKJRPS7h8yqqMyaNQtqtdqwJSYmih2JiIiI6pBZnfqRy+WQy+VixyAiIqJ6YlYrKkRERGRdRF1Ryc3NxY0bNwyv4+PjERsbCzc3NzRs2FDEZERERGQKRC0qp0+fRp8+fQyvp0+fDgAYP3481qxZI1IqIiIiMhWiFpXevXtDEMS97YmIiIhMF69RISIiIpPFokJEREQmi0WFiIiITBaLChEREZksFhUiIiIyWSwqREREZLJYVIiIiMhkmdWzfv6ufAaLRqMROQkRERFVV/nv7erMUjPropKTkwMACAgIEDkJERERGSsnJwcqleqRx0gEMx4Nq9frkZycDGdnZ0gkErHjmCSNRoOAgAAkJiZCqVSKHcfq8edhWvjzMC38eZieuvqZCIKAnJwc+Pr6Qip99FUoZr2iIpVK4e/vL3YMs6BUKvl/fBPCn4dp4c/DtPDnYXrq4mfyuJWUcryYloiIiEwWiwoRERGZLBYVCyeXyzFnzhzI5XKxoxD48zA1/HmYFv48TI8p/EzM+mJaIiIismxcUSEiIiKTxaJCREREJotFhYiIiEwWiwoRERGZLBYVCxQeHo4uXbrA2dkZnp6eGDp0KK5duyZ2LCoTHh4OiUSCadOmiR3FqiUlJWHcuHFwd3eHg4MD2rdvjzNnzogdyyqVlJRg9uzZCA4Ohr29PRo1aoR58+ZBr9eLHc0qHDp0CIMHD4avry8kEgm2bt1a4eOCIGDu3Lnw9fWFvb09evfujUuXLtVbPhYVCxQVFYUpU6bgxIkT2LNnD0pKShAaGoq8vDyxo1m96OhorFq1Cm3bthU7ilXLyspC9+7dYWtrix07duDy5ctYunQpXFxcxI5mlRYtWoRvvvkGK1aswJUrV7B48WIsWbIEy5cvFzuaVcjLy0O7du2wYsWKKj++ePFifP7551ixYgWio6Ph7e2NAQMGGJ63V9d4e7IVSE9Ph6enJ6KiotCzZ0+x41it3NxcdOzYEV9//TU+/fRTtG/fHsuWLRM7llWaOXMmjh49isOHD4sdhQC88MIL8PLywnfffWfYN2LECDg4OOCnn34SMZn1kUgkiIyMxNChQwGUrqb4+vpi2rRp+PDDDwEAWq0WXl5eWLRoESZPnlznmbiiYgXUajUAwM3NTeQk1m3KlCkYNGgQ+vfvL3YUq/f777+jc+fOGDVqFDw9PdGhQwd8++23YseyWj169MC+fftw/fp1AMC5c+dw5MgRPP/88yIno/j4eKSmpiI0NNSwTy6Xo1evXjh27Fi9ZDDrhxLS4wmCgOnTp6NHjx5o3bq12HGs1saNGxETE4Po6GixoxCAW7duYeXKlZg+fTr+9a9/4dSpU3jvvfcgl8vx6quvih3P6nz44YdQq9Vo3rw5ZDIZdDodFixYgDFjxogdzeqlpqYCALy8vCrs9/Lywp07d+olA4uKhXvnnXdw/vx5HDlyROwoVisxMRFTp07F7t27oVAoxI5DAPR6PTp37oyFCxcCADp06IBLly5h5cqVLCoi+Pnnn7F27VqsX78erVq1QmxsLKZNmwZfX1+MHz9e7HiE0lNCDxIEodK+usKiYsHeffdd/P777zh06BD8/f3FjmO1zpw5g7S0NHTq1MmwT6fT4dChQ1ixYgW0Wi1kMpmICa2Pj48PWrZsWWFfixYtsGXLFpESWbcPPvgAM2fOxEsvvQQAaNOmDe7cuYPw8HAWFZF5e3sDKF1Z8fHxMexPS0urtMpSV3iNigUSBAHvvPMOfv31V+zfvx/BwcFiR7Jq/fr1w4ULFxAbG2vYOnfujLFjxyI2NpYlRQTdu3evdMv+9evXERgYKFIi65afnw+ptOKvI5lMxtuTTUBwcDC8vb2xZ88ew76ioiJERUWhW7du9ZKBKyoWaMqUKVi/fj1+++03ODs7G84xqlQq2Nvbi5zO+jg7O1e6PsjR0RHu7u68bkgk77//Prp164aFCxdi9OjROHXqFFatWoVVq1aJHc0qDR48GAsWLEDDhg3RqlUrnD17Fp9//jlee+01saNZhdzcXNy4ccPwOj4+HrGxsXBzc0PDhg0xbdo0LFy4EE2bNkXTpk2xcOFCODg44OWXX66fgAJZHABVbhEREWJHozK9evUSpk6dKnYMq/bHH38IrVu3FuRyudC8eXNh1apVYkeyWhqNRpg6darQsGFDQaFQCI0aNRL+/e9/C1qtVuxoVuHAgQNV/s4YP368IAiCoNfrhTlz5gje3t6CXC4XevbsKVy4cKHe8nGOChEREZksXqNCREREJotFhYiIiEwWiwoRERGZLBYVIiIiMlksKkRERGSyWFSIiIjIZLGoEBERkcliUSEis9G7d29MmzZN7BhEVI9YVIioxiZMmIChQ4dW2Ld582YoFAosXrxYnFBEZFH4rB8iqjWrV6/GlClT8NVXX2HSpElixyEiC8AVFSKqFYsXL8Y777yD9evXVyopu3btgkKhQHZ2doX97733Hnr16gUAuH//PsaMGQN/f384ODigTZs22LBhwyO/pkQiwdatWyvsc3FxwZo1awyvk5KS8OKLL8LV1RXu7u4YMmQIbt++XdNvk4jqGYsKET2xmTNnYv78+di2bRtGjBhR6eP9+/eHi4sLtmzZYtin0+nwyy+/YOzYsQCAwsJCdOrUCdu2bcPFixfxf//3f3jllVdw8uTJGufKz89Hnz594OTkhEOHDuHIkSNwcnLCwIEDUVRUVOP3JaL6w1M/RPREduzYgd9++w379u1D3759qzxGJpPhxRdfxPr16/H6668DAPbt24esrCyMGjUKAODn54cZM2YYPufdd9/Fzp07sWnTJjz99NM1yrZx40ZIpVKsXr0aEokEABAREQEXFxccPHgQoaGhNXpfIqo/XFEhoifStm1bBAUF4eOPP0ZOTg4AICwsDE5OTnByckKrVq0AAGPHjsXBgweRnJwMAFi3bh2ef/55uLq6AihdYVmwYAHatm0Ld3d3ODk5Yffu3UhISKhxtjNnzuDGjRtwdnY25HFzc0NhYSFu3rz5hN85EdUHrqgQ0RPx8/PDli1b0KdPHwwcOBA7d+7E6tWrUVBQAACwtbUFADz11FNo3LgxNm7ciLfeeguRkZGIiIgwvM/SpUvxxRdfYNmyZWjTpg0cHR0xbdq0R56ikUgkEAShwr7i4mLDn/V6PTp16oR169ZV+lwPD48n+r6JqH6wqBDRE2vYsCGioqLQp08fhIaGYteuXfDz86t03Msvv4x169bB398fUqkUgwYNMnzs8OHDGDJkCMaNGwegtGTExcWhRYsWD/26Hh4eSElJMbyOi4tDfn6+4XXHjh3x888/w9PTE0qlsja+VSKqZzz1Q0S1wt/fHwcPHsT9+/cRGhoKtVpd6ZixY8ciJiYGCxYswMiRI6FQKAwfa9KkCfbs2YNjx47hypUrmDx5MlJTUx/5Nfv27YsVK1YgJiYGp0+fxptvvmlYwSn/eg0aNMCQIUNw+PBhxMfHIyoqClOnTsXdu3dr75snojrDokJEtcbPzw9RUVHIzs7GgAEDKt2O3LRpU3Tp0gXnz5833O1T7qOPPkLHjh3x3HPPoXfv3vD29q40TO7vli5dioCAAPTs2RMvv/wyZsyYAQcHB8PHHRwccOjQITRs2BDDhw9HixYt8Nprr6GgoIArLERmQiL8/QQvERERkYngigoRERGZLBYVIiIiMlksKkRERGSyWFSIiIjIZLGoEBERkcliUSEiIiKTxaJCREREJotFhYiIiEwWiwoRERGZLBYVIiIiMlksKkRERGSyWFSIiIjIZP1/cB70fGhjz5QAAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Display the Elbow Curve graph.\n",
        "\n",
        "plt.plot(range(1, 11), inertias)\n",
        "plt.xlabel('K-value')\n",
        "plt.ylabel('Inertia Score')\n",
        "plt.title('Elbow Curve')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zx0X3uRmYFTA",
        "outputId": "78965ed7-1fa5-41c1-eba8-f2c2eeef524b"
      },
      "outputs": [
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAHFCAYAAADmGm0KAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAByyUlEQVR4nO3dd3hTZfsH8O9J0qZ77z3YlNlCoYgMsQq8ICpTnODrQgUBf4o4EUFReVEUVLaKgDIUlVWRvfew7BY6aCktnXQn5/dHmkBtWpqS9mR8P9eV6317cnLOnVaau89zP/cjiKIogoiIiIiqkUkdABEREZEpYpJEREREpAeTJCIiIiI9mCQRERER6cEkiYiIiEgPJklEREREejBJIiIiItKDSRIRERGRHkySiIiIiPRgkkRkxr788ksIgoCoqCipQzE7qampeOmll9CiRQvY29vDw8MD7dq1w3//+1+kpqZKHV6DJCUl4eWXX9a9JwcHB7Rt2xZvv/020tPTpQ6PyOwI3JaEyHx17NgRJ06cAADs378fsbGxEkdkHtLS0tCpUye4ublh0qRJaNmyJfLz85GYmIiff/4ZX3zxBXr16iV1mAb5448/MHLkSHh5eeHll19Gp06dIAgCTp06hcWLF0Mmk+HYsWNSh0lkVpgkEZmpw4cPo0uXLhg4cCD+/PNP/Pe//8V3330ndVh6FRcXw8HBQeowdN577z1MmzYNSUlJCA8Pr/G8Wq2GTNY0A+0lJSWws7ODIAgNvkZycjLatWuHFi1aYNu2bXB1da32vCiKWLduHR555JG7DRcVFRUQBAEKheKur0Vk6jjdRmSmFi1aBAD4+OOPERcXh5UrV6K4uLjGeenp6XjuuecQHBwMW1tbBAQEYOjQobh27ZrunLy8PEyaNAkRERFQKpXw8fHBgAEDcPbsWQDA9u3bIQgCtm/fXu3aly9fhiAIWLp0qe7Y008/DScnJ5w6dQrx8fFwdnbGfffdBwBISEjAQw89hKCgINjZ2aFZs2Z4/vnnkZ2dXSPus2fPYtSoUfD19YVSqURISAiefPJJlJWV4fLly1AoFJg5c2aN1+3cuROCIOCXX36p9XuXk5MDmUwGHx8fvc//O0E6cOAABg0aBE9PT9jZ2SEyMhITJkyods7u3btx3333wdnZGQ4ODoiLi8Off/5Z7ZylS5dCEARs2bIFY8aMgbe3NxwcHFBWVgYAWLVqFbp37w5HR0c4OTnhgQceqNfoz+zZs3Hz5k3MmzevRoIEAIIgVEuQwsLC8PTTT9c4r3fv3ujdu7fua+3P/YcffsCkSZMQGBgIpVKJf/75B4Ig6P4bvN3GjRshCALWr1+vO3bhwgU89thj8PHxgVKpROvWrfH111/f8X0RSY1JEpEZKikpwYoVK9ClSxdERUVhzJgxKCwsrJEYpKeno0uXLli3bh0mTpyIjRs3Ys6cOXB1dUVubi4AoLCwEPfccw++/fZbPPPMM/j999/xzTffoEWLFsjIyGhQfOXl5Rg8eDD69u2L3377DR988AEA4NKlS+jevTvmz5+PLVu24N1338WBAwdwzz33oKKiQvf6EydOoEuXLti/fz+mTZuGjRs3YubMmSgrK0N5eTnCwsIwePBgfPPNN1CpVNXu/dVXXyEgIAAPP/xwrfF1794darUajzzyCDZv3oyCgoJaz928eTN69uyJlJQUzJ49Gxs3bsTbb79dLcncsWMH+vbti/z8fCxatAgrVqyAs7MzBg0ahFWrVtW45pgxY2BjY4MffvgBq1evho2NDWbMmIFRo0ahTZs2+Pnnn/HDDz+gsLAQPXv2RGJiYp3f7y1btsDX1xfdunWr87yGmjJlClJSUvDNN9/g999/R3BwMDp16oQlS5bUOHfp0qW6JBsAEhMT0aVLF5w+fRqff/45/vjjDwwcOBCvvvqq7r8LIpMlEpHZ+f7770UA4jfffCOKoigWFhaKTk5OYs+ePaudN2bMGNHGxkZMTEys9VrTpk0TAYgJCQm1nrNt2zYRgLht27Zqx5OTk0UA4pIlS3THnnrqKRGAuHjx4jrfg1qtFisqKsQrV66IAMTffvtN91zfvn1FNzc3MSsr644xrVu3TncsPT1dVCgU4gcffHDHez///POiTCYTAYiCIIitW7cWX3vtNTE5ObnauZGRkWJkZKRYUlJS6/W6desm+vj4iIWFhbpjlZWVYlRUlBgUFCSq1WpRFEVxyZIlIgDxySefrPb6lJQUUaFQiK+88kq144WFhaKfn584fPjwOt+PnZ2d2K1btzrPuV1oaKj41FNP1Tjeq1cvsVevXrqvtd/je++9t8a5X375pQhAPHfunO7YjRs3RKVSKU6aNEl37IEHHhCDgoLE/Pz8aq9/+eWXRTs7O/HGjRv1jpuoqXEkicgMLVq0CPb29hg5ciQAwMnJCcOGDcOuXbtw4cIF3XkbN25Enz590Lp161qvtXHjRrRo0QL9+vUzaoyPPvpojWNZWVl44YUXEBwcDIVCARsbG4SGhgIAzpw5A0BTv7Rjxw4MHz4c3t7etV6/d+/e6NChQ7Vpm2+++QaCIOC5556rMzZBEPDNN98gKSkJ8+bNwzPPPIOKigr873//Q9u2bbFjxw4AwPnz53Hp0iWMHTsWdnZ2eq918+ZNHDhwAEOHDoWTk5PuuFwuxxNPPIG0tDScO3euzu/N5s2bUVlZiSeffBKVlZW6h52dHXr16lVjmrOp6ftZjh49GkqlstpU64oVK1BWVoZnnnkGAFBaWoqtW7fi4YcfhoODQ7X3NmDAAJSWlmL//v1N9TaIDMYkicjMXLx4ETt37sTAgQMhiiLy8vKQl5eHoUOHAgAWL16sO/f69esICgqq83r1OcdQDg4OcHFxqXZMrVYjPj4ea9euxf/93/9h69atOHjwoO5DsqSkBACQm5sLlUpVr5heffVVbN26FefOnUNFRQUWLFiAoUOHws/Pr15xhoaG4sUXX8SiRYtw4cIFrFq1CqWlpXj99dcBaL43AOqMJTc3F6Iowt/fv8ZzAQEBADQ1ULf797naqbsuXbrAxsam2mPVqlV6a7ZuFxISguTk5Du824bT9948PDwwePBgfP/997opz6VLl6Jr165o27YtAM37rqysxNy5c2u8L+103J3eG5GUuDyByMwsXrwYoihi9erVWL16dY3nly1bhunTp0Mul8Pb2xtpaWl1Xq8+52hHUbQFxlq1fcDpW6l1+vRpnDhxAkuXLsVTTz2lO37x4sVq53l4eEAul98xJgB47LHH8MYbb+Drr79Gt27dkJmZiXHjxt3xdbUZPnw4Zs6cidOnTwOAbiSrrljc3d0hk8n01m9dvXoVAODl5VXt+L+/P9rnV69erRtZM8QDDzyAuXPnYv/+/fWqS7Kzs6vxswQ0P89/x6ovXq1nnnkGv/zyCxISEhASEoJDhw5h/vz5uufd3d11I2q1/Vz0rS4kMhUcSSIyIyqVCsuWLUNkZCS2bdtW4zFp0iRkZGRg48aNAID+/ftj27ZtNaZ7bte/f3+cP38ef//9d63nhIWFAQBOnjxZ7fjtK5juRPtBq1Qqqx3/9ttvq31tb2+PXr164ZdffrnjKIOdnR2ee+45LFu2DLNnz0bHjh3Ro0ePO8ZSW0F6UVERUlNTdSNALVq0QGRkJBYvXqw3qQAAR0dHxMbGYu3atbrRMEAzcvbjjz8iKCgILVq0qDOeBx54AAqFApcuXUJMTIzeR11ee+01ODo64qWXXkJ+fn6N58WqFgBaYWFhNX6W58+fr/O/E33i4+MRGBiIJUuWYMmSJbCzs8OoUaN0zzs4OKBPnz44duwY2rdvr/d9eXp6GnRPoiYlbUkUERni999/FwGIn3zyid7nr1+/LiqVSnHIkCGiKIpiWlqa6O/vL/r4+Ihz5swRt27dKq5Zs0b873//K545c0YURVEsKCgQ27ZtKzo5OYnTp08Xt2zZIv7222/ixIkTxb///lt37X79+onu7u7iggULxC1btohvvPGG2Lx5c72F246OjjViKy8vFyMjI8XQ0FDxp59+Ejdt2iSOGzdObNGihQhAfO+993TnHj9+XHRychIjIiLE7777Tvz777/FFStWiKNGjRILCgqqXTctLU1UKBQiAHHhwoX1+j6OGzdO7Nixozhz5kxx48aN4vbt28UlS5aI0dHRNYrON23aJNrY2IgdO3YUly1bJm7btk1ctmyZ+Nhjj+nO2b59u2hjYyPGxsaKv/zyi/jbb7+JDzzwgCgIgrhy5UrdedrC7UOHDtWIacaMGaJCoRCff/55cd26deL27dvFVatWiZMmTRLffffdO76n33//XXRwcBDDwsLEzz77TNy6dau4detWce7cuWKnTp3Ejh076s798ccfRQDiiy++KP7111/iokWLxJYtW4r+/v56C7d/+eWXWu87ZcoUUalUit7e3tW+J1r//POP6O7uLnbt2lVcsmSJuG3bNnH9+vXi7NmzxT59+tzxfRFJiUkSkRkZMmSIaGtrW+eqr5EjR4oKhULMzMwURVEUU1NTxTFjxoh+fn6ijY2NGBAQIA4fPly8du2a7jW5ubni+PHjxZCQENHGxkb08fERBw4cKJ49e1Z3TkZGhjh06FDRw8NDdHV1FR9//HHx8OHD9U6SRFEUExMTxfvvv190dnYW3d3dxWHDhokpKSk1kiTtucOGDRM9PT1FW1tbMSQkRHz66afF0tLSGtft3bu36OHhIRYXF9fn2yju379fHDdunNihQwfRw8NDlMvlore3t/jggw+KGzZsqHH+vn37xP79+4uurq6iUqkUIyMjxddee63aObt27RL79u0rOjo6ivb29mK3bt3E33//vdo5dSVJoiiKv/76q9inTx/RxcVFVCqVYmhoqDh06FDxr7/+qtf7unTpkvjSSy+JzZo1E5VKpWhvby+2adNGnDhxYrVVe2q1Wpw1a5YYEREh2tnZiTExMeLff/9d6+q2upKk8+fPiwDqXCGZnJwsjhkzRgwMDBRtbGxEb29vMS4uTpw+fXq93heRVNhxm4jMWlZWFkJDQ/HKK69g1qxZUodDRBaEhdtEZJbS0tKQlJSETz/9FDKZDOPHj5c6JCKyMCzcJiKztHDhQvTu3Rv//PMPli9fjsDAQKlDIiILw+k2IiIiIj04kkRERESkB5MkIiIiIj2YJBERERHpwdVtDaRWq3H16lU4OzvX2rKfiIiITIsoiigsLERAQABksrrHipgkNdDVq1cRHBwsdRhERETUAKmpqXfcSJtJUgM5OzsD0HyT/73bOREREZmmgoICBAcH6z7H68IkqYG0U2wuLi5MkoiIiMxMfUplWLhNREREpAeTJCIiIiI9mCQRERER6cEkiYiIiEgPJklEREREejBJIiIiItKDSRIRERGRHkySiIiIiPRgkkRERESkB5MkIiIiIj2YJBERERHpwSSJiIiISA8mSdRoSspVEEVR6jCIiIgahEkSNYqM/BLETE/AoK92Iy23WOpwiIiIDMYkiRrF4cu5uFmuwun0Agz+ag8OJOVIHRIREZFBmCRRo0i6fhMAIAjAjZvlGL3wAH7cf0XiqIiIiOqPSRI1iuTsIgDAK32b4z/t/VGpFvH2r6cxdd0plFeqJY6OiIjozpgkUaNIztaMJLXxd8HcUZ3w+gMtIQjA8gMpeHzRAeQUlUkcIRERUd2YJJHRiaKIpKokKcLbEYIgYFyfZlj4ZAyclAocTL6BwV/twT9X8yWOlIiIqHZMksjocm6Wo7C0EoIAhHg46I7f19oXv46LQ5inA9LzSjB0/j78eTJDwkiJiIhqxySJjE471Rbkbg87G3m155r5OOO3cfegZ3MvlFSoMO6no/h8yzmo1eynREREpoVJEhld0nVN0Xa4l5Pe510dbLDk6S549p5wAMDcvy/i+R+PoKisssliJCIiuhMmSWR0unokL8daz1HIZXj7P23w+bAOsFXIkJB4DY/M24MrOTebKkwiIqI6MUkio0uu6pEUXkeSpPVodBBWPdcNPs5KnL9WhMFf7cHuC9mNHSIREdEdMUkio9PWJNUnSQKATiHu+P2Ve9Ah2A35JRV4aslBLN6dzH3fiIhIUkySyKhUahFXcjR7tdU3SQIAXxc7rHquGx7pHAiVWsS0PxLxf6tPoqxS1VihEhER1YlJEhlVem4JylVq2CpkCHSzN+i1djZyfD6sA94e2BoyAfjlSBpGfbcfWYWljRQtERFR7ZgkkVElVW1HEu7pCJlMMPj1giDg2Z4RWPJMV7jYKXA0JQ+D5+7BybQ8I0dKRERUNyZJZFSG1iPVplcLb/z28j1o5uOEzIJSDPtmH349lm6MEImIiOqFSRIZlS5J8r67JAnQJFrrXorDfa18UFapxoRVxzFzwxmo2HiSiIiaAJMkMipjjSRpOdvZ4LsnY/BS70gAwLc7kzB22SHkl1QY5fpERES1YZJERpV0/c6NJA0llwn4vwdbYe6oTrCzkWH7uet4+Os9uFTV2ZuIiKgxMEkioymtUOFqfgkAIMJb/5Ykd2NQhwCsfiEOAa52SMq+iSFf78G2c1lGvw8RERHAJImM6HLOTYgi4GpvA3cHm0a5R1SgK357+R7EhLqjsLQSY5Yewrc7LrHxJBERGR2TJDKa27cjEQTDl//Xl7ezEj/9txtGdQ2GKAIzN57Fa6uOo7SCjSeJiMh4mCSR0dRnY1tjsVXIMOPhdpj2UFvIZQJ+PX4Vw7/dh4yq6T4iIqK7xSSJjMbYK9vuRBAEPNk9DD+OjYW7gw1OpuVj0Nw9OHLlRpPcn4iILBuTJDIaY/ZIMkT3SE+sf/ketPJzRnZRGUZ9dwA/H0pt0hiIiMjyMEkio0mqWpIf4WX8lW13EuzhgDUvxuHBtn4oV6nxf2tO4oPf/0GlSt3ksRARkWWQPEmaN28ewsPDYWdnh+joaOzatavWc59++mkIglDj0bZtW905CxYsQM+ePeHu7g53d3f069cPBw8erHad999/v8Y1/Pz8Gu09WoPcm+XILdY0eAzzcpAkBkelAvNGd8aEfs0BAEv2XMZTSw4i92a5JPEQEZF5kzRJWrVqFSZMmICpU6fi2LFj6NmzJ/r374+UlBS953/xxRfIyMjQPVJTU+Hh4YFhw4bpztm+fTtGjRqFbdu2Yd++fQgJCUF8fDzS06vv+9W2bdtq1zp16lSjvldLl5yjmWrzd7WDg61CsjhkMgET+rXAN493hoOtHHsu5uChr/fg/LVCyWIiIiLzJGmSNHv2bIwdOxbPPvssWrdujTlz5iA4OBjz58/Xe76rqyv8/Px0j8OHDyM3NxfPPPOM7pzly5fjpZdeQseOHdGqVSssWLAAarUaW7durXYthUJR7Vre3t6N+l4t3e3L/03Bg1H+WPtSHII97JFyoxgPf70HW/7JlDosIiIyI5IlSeXl5Thy5Aji4+OrHY+Pj8fevXvrdY1FixahX79+CA0NrfWc4uJiVFRUwMPDo9rxCxcuICAgAOHh4Rg5ciSSkpLqvFdZWRkKCgqqPeiWpl7ZVh+t/Fzw27h70D3CEzfLVXjuhyOYu/UCG08SEVG9SJYkZWdnQ6VSwdfXt9pxX19fZGbe+S/+jIwMbNy4Ec8++2yd57355psIDAxEv379dMdiY2Px/fffY/PmzViwYAEyMzMRFxeHnJycWq8zc+ZMuLq66h7BwcF3jNGamGKSBAAejrb4fmxXPNVdk0h/nnAeL/90DMXllRJHRkREpk7ywu1/d2YWRbFe3ZqXLl0KNzc3DBkypNZzZs2ahRUrVmDt2rWws7PTHe/fvz8effRRtGvXDv369cOff/4JAFi2bFmt15oyZQry8/N1j9RULjG/nXaz2chG2LPtbtnIZfjgoSh8/Eg72MgF/HkqA4/O34e03GKpQyMiIhMmWZLk5eUFuVxeY9QoKyurxujSv4miiMWLF+OJJ56Ara2t3nM+++wzzJgxA1u2bEH79u3rvJ6joyPatWuHCxcu1HqOUqmEi4tLtQdpqNUiLueY5kjS7UZ2DcGK/3aDl5MtzmQUYPBXe3AgqfbRQyIism6SJUm2traIjo5GQkJCteMJCQmIi4ur87U7duzAxYsXMXbsWL3Pf/rpp/jwww+xadMmxMTE3DGWsrIynDlzBv7+/vV/A6STWVCK0go1FDIBQe72UodTp5gwD/z28j2ICnTBjZvlGL3wAH7cf0XqsIiIyARJOt02ceJELFy4EIsXL8aZM2fw2muvISUlBS+88AIAzRTXk08+WeN1ixYtQmxsLKKiomo8N2vWLLz99ttYvHgxwsLCkJmZiczMTBQVFenOmTx5Mnbs2IHk5GQcOHAAQ4cORUFBAZ566qnGe7MWTFuPFOLpAIVc8hncOwp0s8cvz8dhUIcAVKpFvP3raby17hTKK9l4koiIbpGuoQ2AESNGICcnB9OmTUNGRgaioqKwYcMG3Wq1jIyMGj2T8vPzsWbNGnzxxRd6rzlv3jyUl5dj6NCh1Y6/9957eP/99wEAaWlpGDVqFLKzs+Ht7Y1u3bph//79da6So9o15ca2xmJvK8eXIzuitb8zPt18Dj8dSMHFa0WY/3hneDoppQ6PiIhMgCByPXSDFBQUwNXVFfn5+VZfnzTt90Qs3pOM5+6NwFsDWksdjsH+PnsN41ccR2FZJQLd7PHdk9FoG+AqdVhERNQIDPn8Nv25ETJ5SdmaqUxTLtquS99Wvlg3Lg7hXo5IzyvB0Pn78OfJDKnDIiIiiTFJortmqj2SDNHMxxm/vtQD97bwRkmFCuN+OorPt5yDWs2BViIia8Ukie5KeaUaqTc0/YbMqSZJH1cHGyx+Kgb/7RkOAJj790U898MRFJZWSBwZERFJgUkS3ZWUG8VQi4CjrRzezuZf8KyQyzB1YBt8PqwDbBUy/HXmGh6ZtxdXqvpAERGR9WCSRHdFN9Xm7VivTunm4tHoIPz8fHf4OCtxIasIo77bz61MiIisDJMkuitJVduRRHiZ3nYkd6tjsBt+f+UeBLnb42p+Kb7ZUfcmyEREZFmYJNFdsYSi7br4uthhalVbg293XEJ6XonEERERUVNhkkR3RddI0tsykyQAeDDKD13DPVBWqcYnG89KHQ4RETURJkl0Vyx9JAkABEHAu/9pA0EA1p+4iiNXbkgdEhERNQEmSdRghaUVuF5YBgAIs+AkCQCiAl0xPDoYADDtjzPsn0REZAWYJFGDXc7W9EfyclLCxc5G4mga36QHWsBJqcCJ1Dz8ejxd6nCIiKiRMUmiBtNuR2LJ9Ui383G2w7g+zQAAn2w6y5YAREQWjkkSNVjS9aqibQufarvdMz3CEOxhj2sFZWwJQERk4ZgkUYNZQ9H2v9nZyPFWf7YEICKyBkySqMGsMUkC2BKAiMhaMEmiBhFFUZckWUtNkhZbAhARWQcmSdQg14vKUFRWCZkABHs4SB1Ok2NLACIiy8ckiRpEW7Qd7OEApUIucTTSYEsAIiLLxiSJGsRa65Fux5YARESWjUkSNQiTJA22BCAislxMkqhBrLFHkj5sCUBEZLmYJFGDJFd12w73cpI4EumxJQARkWVikkQGq1SpkXJDs29buJUt/9eHLQGIiCwTkyQyWFpuCSpUIuxsZPB3sZM6HJPAlgBERJaHSRIZTFu0HebpCJlMkDga08GWAEREloVJEhksyUo7bd8JWwIQEVkWJklksFtF20yS/o0tAYiILAeTJDLYrR5JXNn2b2wJQERkOZgkkcGSr7ORZF3YEoCIyDIwSSKDFJdX4mp+KQAgkjVJerElABGRZWCSRAa5nK3pj+TuYAM3B1uJozFdbAlARGT+mCSRQbhnW/2xJQARkXljkkQG4XYk9ceWAERE5o1JEhmEPZIMw5YARETmi0kSGUQ73RbB6bZ6YUsAIiLzxSSJ6k0URSRpl/9zJKne2BKAiMg8SZ4kzZs3D+Hh4bCzs0N0dDR27dpV67lPP/00BEGo8Wjbtm2189asWYM2bdpAqVSiTZs2WLdu3V3dlzRyiyuQX1IBQLNvG9UPWwIQEZknSZOkVatWYcKECZg6dSqOHTuGnj17on///khJSdF7/hdffIGMjAzdIzU1FR4eHhg2bJjunH379mHEiBF44okncOLECTzxxBMYPnw4Dhw40OD7koa2aDvQzR52NnKJozEv1VoC/J7IlgBERGZAEEVRst/WsbGx6Ny5M+bPn6871rp1awwZMgQzZ8684+t//fVXPPLII0hOTkZoaCgAYMSIESgoKMDGjRt15z344INwd3fHihUrjHJfACgoKICrqyvy8/Ph4uJSr9eYu18Op+L11SdxTzMv/PhsrNThmJ2swlL0/WwHisoqMXt4BzzSOUjqkIiIrI4hn9+SjSSVl5fjyJEjiI+Pr3Y8Pj4ee/furdc1Fi1ahH79+ukSJEAzkvTvaz7wwAO6azb0vmVlZSgoKKj2sDbskXR32BKAiMi8SJYkZWdnQ6VSwdfXt9pxX19fZGZm3vH1GRkZ2LhxI5599tlqxzMzM+u8ZkPvO3PmTLi6uuoewcHBd4zR0miLtrn8v+HYEoCIyHxIXrgtCEK1r0VRrHFMn6VLl8LNzQ1Dhgxp0DUNve+UKVOQn5+ve6Smpt4xRkvDkaS7x5YARETmQ7IkycvLC3K5vMboTVZWVo1Rnn8TRRGLFy/GE088AVvb6vuH+fn51XnNht5XqVTCxcWl2sOaqNUiknO0PZLYbftusCUAEZF5kCxJsrW1RXR0NBISEqodT0hIQFxcXJ2v3bFjBy5evIixY8fWeK579+41rrllyxbdNe/mvtbsan4JyivVsJELCHS3lzocs8aWAERE5kHS6baJEydi4cKFWLx4Mc6cOYPXXnsNKSkpeOGFFwBopriefPLJGq9btGgRYmNjERUVVeO58ePHY8uWLfjkk09w9uxZfPLJJ/jrr78wYcKEet+XatJOtYV6OkIuu/N0KNWNLQGIiEyfQsqbjxgxAjk5OZg2bRoyMjIQFRWFDRs26FarZWRk1OhdlJ+fjzVr1uCLL77Qe824uDisXLkSb7/9Nt555x1ERkZi1apViI29tWT9TvelmliPZHyTHmiBP09l4ERaPn49ns6WAEREJkbSPknmzNr6JL2//h8s3XsZz/eKwJSqwmO6e/O3X8Inm87C10WJbZN7w8FW0r9biIgsnln0SSLzksSNbRsFWwIQEZkuJklUL9otScK5ss2o2BKAiMh0MUmiOyqrVCEtV/PhzZok42NLACIi08Qkie4oJacYogg4KxXwcrK98wvIIGwJQERkmpgk0R1p65HCvR3r1Q2dDMeWAEREpodJEt2Rbs82TrU1qkkPtICTUqFrCUBERNJikkR3xKLtpuHjbIdxfZoBAD7ZdBbF5ZUSR0REZN2YJNEdJd823UaNq1pLgO2XpA6HiMiqMUmiO0pmj6QmU60lwM4ktgQgIpIQkySqU35JBbKLygEAYUySmsSDUX6IZUsAIiLJMUmiOl2uGkXycVbCScktM5qCIAh4hy0BiIgkxySJ6pRUVbQdwXqkJsWWAERE0mOSRHVKrlr+z5VtTY8tAYiIpMUkierEjW2lw5YARETSYpJEddIt/2eSJAm2BCAikg6TJKqVKIrskSQxtgQgIpIOkySqVVZhGYrLVZDLBIR4OEgdjtViSwAiImkwSaJaXbquWdkW4uEAGzn/U5EKWwIQEUmDn3xUK9YjmQ62BCAianpMkqhWt5b/M0kyBWwJQETUtJgkUa04kmRa2BKAiKhpMUmiWnFjW9PDlgBERE2HSRLpVaFSI+VGMQAgwpvdtk0FWwIQETUdJkmkV+qNYlSqRdjbyOHropQ6HLoNWwIQETUNJkmk1+31SIIgSBwN3Y4tAYiImgaTJNKLnbZNG1sCEBE1PiZJpBc3tjV9bAlARNS4mCSRXuyRZPrYEoCIqHExSSK9krI1W5JwZZtpY0sAIqLGwySJarhZVolrBWUAgHBPjiSZMrYEICJqPEySqAZt0banoy1cHWwkjobuhC0BiIgaB5MkqoHbkZgXtgQgImocTJKoBiZJ5octAYiIjO+ukqTS0lJjxUEmhD2SzBNbAhARGZfBSZJarcaHH36IwMBAODk5ISkpCQDwzjvvYNGiRUYPkJpe0vWqlW1eXNlmTv7dEuBmGVsCEBHdDYOTpOnTp2Pp0qWYNWsWbG1tdcfbtWuHhQsXGjU4anqiKN5qJMmRJLNze0uAb3ewJQAR0d0wOEn6/vvv8d1332H06NGQy+W64+3bt8fZs4avrJk3bx7Cw8NhZ2eH6Oho7Nq1q87zy8rKMHXqVISGhkKpVCIyMhKLFy/WPd+7d28IglDjMXDgQN0577//fo3n/fz8DI7dEuXcLEdhaSUEAQjxcJA6HDIQWwIQERmPwtAXpKeno1mzZjWOq9VqVFRUGHStVatWYcKECZg3bx569OiBb7/9Fv3790diYiJCQkL0vmb48OG4du0aFi1ahGbNmiErKwuVlbemFdauXYvy8nLd1zk5OejQoQOGDRtW7Tpt27bFX3/9pfv69oTPmmnrkQLd7GFnw++JOdK2BDiQfAOfbDyLL0d1kjokIiKzZHCS1LZtW+zatQuhoaHVjv/yyy/o1MmwX8azZ8/G2LFj8eyzzwIA5syZg82bN2P+/PmYOXNmjfM3bdqEHTt2ICkpCR4eHgCAsLCwaudoj2utXLkSDg4ONZIkhULB0SM9uB2J+dO2BBj01W6sP3EVT8WFIjrU484vJCKiagyebnvvvffw8ssv45NPPoFarcbatWvx3//+FzNmzMC7775b7+uUl5fjyJEjiI+Pr3Y8Pj4ee/fu1fua9evXIyYmBrNmzUJgYCBatGiByZMno6Sk9imFRYsWYeTIkXB0rP6hf+HCBQQEBCA8PBwjR47UFaDXpqysDAUFBdUelogb21oGtgQgIrp7BidJgwYNwqpVq7BhwwYIgoB3330XZ86cwe+//47777+/3tfJzs6GSqWCr69vteO+vr7IzMzU+5qkpCTs3r0bp0+fxrp16zBnzhysXr0a48aN03v+wYMHcfr0ad1IlVZsbCy+//57bN68GQsWLEBmZibi4uKQk5NTa7wzZ86Eq6ur7hEcHFzv92pOdCvbuGeb2WNLACKiu2NQklRZWYkPPvgAbdq0wY4dO1BUVITi4mLs3r27xohQfQmCUO1rURRrHNNSq9UQBAHLly9H165dMWDAAMyePRtLly7VO5q0aNEiREVFoWvXrtWO9+/fH48++ijatWuHfv364c8//wQALFu2rNY4p0yZgvz8fN0jNTXV0LdqFthI0nKwJQAR0d0xKElSKBT49NNPoVKp7vrGXl5ekMvlNUaNsrKyaowuafn7+yMwMBCurq66Y61bt4YoikhLS6t2bnFxMVauXFljFEkfR0dHtGvXDhcuXKj1HKVSCRcXl2oPS6NSi7iSUwyASZKlYEsAIqKGM3i6rV+/fti+fftd39jW1hbR0dFISEiodjwhIQFxcXF6X9OjRw9cvXoVRUVFumPnz5+HTCZDUFBQtXN//vlnlJWV4fHHH79jLGVlZThz5gz8/f0b8E4sx9W8EpSr1LBVyBDgZi91OGQEbAlARNRwBq9u69+/P6ZMmYLTp08jOjq6RkH04MGD632tiRMn4oknnkBMTAy6d++O7777DikpKXjhhRcAaKa40tPT8f333wMAHnvsMXz44Yd45pln8MEHHyA7Oxuvv/46xowZA3v76h/qixYtwpAhQ+Dp6VnjvpMnT8agQYMQEhKCrKwsTJ8+HQUFBXjqqacM/XZYFG3RdpinA+Qy/VOeZH5ubwnw8cazmMuWAERE9WJwkvTiiy8C0Czf/zdBEAyaihsxYgRycnIwbdo0ZGRkICoqChs2bNC1F8jIyEBKSorufCcnJyQkJOCVV15BTEwMPD09MXz4cEyfPr3adc+fP4/du3djy5Yteu+blpaGUaNGITs7G97e3ujWrRv2799fo62BtUnmdiQWSdsS4D9zd+P3E1fx3qA28HJSSh0WEZHJE0RR5NrgBigoKICrqyvy8/Mtpj7p3d9O4/t9V/Bi70i88WArqcMhI3twzk6czSzE/NGd0b+ddU8tE5H1MuTz2+CaJLJcXNlm2bqEaRpKHki+IXEkRETmoUFJ0o4dOzBo0CA0a9YMzZs3x+DBg++45xqZvqTrbCRpybqGa5KkQ5eZJBER1YfBSdKPP/6Ifv36wcHBAa+++ipefvll2Nvb47777sNPP/3UGDFSEyitUOFqvmblE0eSLJM2SUrMKEBBqWH7LBIRWSODC7c/+ugjzJo1C6+99pru2Pjx4zF79mx8+OGHeOyxx4waIDWNKznFEEXAxU4BD0dbqcOhRuDrYocwTwdczinGkcu56NPKR+qQiIhMmsEjSUlJSRg0aFCN44MHD0ZycrJRgqKml5x9azuS2jqek/nTjiaxLomI6M4MTpKCg4OxdevWGse3bt1qsfuZWYNLrEeyCtri7YPJte9TSEREGgZPt02aNAmvvvoqjh8/jri4OAiCgN27d2Pp0qX44osvGiNGagJc2WYdYsM1zVVPpuWjpFwFe1u5xBEREZmuBjWT9PPzw+eff46ff/4ZgGb/tFWrVuGhhx4yeoDUNHRJkjeTJEsW7GEPPxc7ZBaU4lhqLuIivaQOiYjIZBmcJAHAww8/jIcfftjYsZCEOJJkHQRBQNdwD6w/cRUHk28wSSIiqoPBNUmHDh3CgQMHahw/cOAADh8+bJSgqGnlFZfjxs1yAEySrIG2ePsgi7eJiOpkcJI0btw4pKam1jienp6OcePGGSUoalraUSR/Vzs42DZocJHMSGxVknQ0JRfllWqJoyEiMl0GfyImJiaic+fONY536tQJiYmJRgmKmpa20zZHkaxDMx8nuDvYILe4AqfS8xEd6i51SFapQqVG0vWbkMsAhUwGhVyAjVwGhUyAjUIGm6pjCpnAthxEEjE4SVIqlbh27RoiIiKqHc/IyIBCwVEIc8R6JOsiCAK6hHlgS+I1HLp8g0mSRJ77/jC2nbter3MVMkGXROkSKfmtJEp3XC7cSq7kMtjKhRoJmO74v87XXENzvo32+X9f+7br2cpl1c6RCYBMJkAmCJr/LwiQyQTIq74WBAFymVDtPLkgQKg6V/scE0IyJQZnNffffz+mTJmC3377Da6urgCAvLw8vPXWW7j//vuNHiA1PiZJ1qdruCZJOph8Ay/0ipQ6HKtTUFqBHec1CZKbgw0qVSIqVGpUqkWo1GKN8yvVIirVIkorLH96VJs06RKtqgRKEFCVSOl/rtp5Vc/JZNUTNtltz1W7XtVzTkoFpgxojUA3e6m/DWQiDE6SPv/8c9x7770IDQ1Fp06dAADHjx+Hr68vfvjhB6MHSI0vqSpJiuDyf6uh7Zd06PINqNQi5DL+9d6UDibdgFrU/GGybXLvas+pqxKiCpVakzyp1bokSptIaZ+rVKtRXqn539sTLc25IipValSoq/5Xd0xzfoX2/NvO0dxPREWlWndO9XNv3aNSddtxtQi1KEKtFiGKgErUfg3N/4oi9OR+emlfr9J8ZeTv/J1dLyzDiv92g4z/JggNSJICAwNx8uRJLF++HCdOnIC9vT2eeeYZjBo1CjY2No0RIzUitVrEZd1IkpPE0VBTae3vDCelAoWllTibWYC2Aa5Sh2RV9lzKBgDERXrWeE4mE2ArE2CrMHhdjUkTxeoJlCgCKnXNZEr1r+eqnVeVbKlFzYhb9eegS9S016jzvNueU4siyirVmPZ7Ig4k38BPB1PweLdQqb9lZAIaVETk6OiI5557ztixkAQyC0pRUqGCQiYg2J1DzNZCIZehc6g7dp6/joPJN5gkNbF9lzTbwlhTnypBW38E0x2hKa1Q4YPfEzFzwxn0aeXDaTeqfwuAixcv4siRI9WObd26FX369EHXrl0xY8YMowdHjU9bjxTi6QCF3LL+cqW6xbJfkiSyi8pwNrMQANBdz0gSSeep7mGIDnXHzXIV3lp7CqLY9NN9ZFrq/an4+uuv49dff9V9nZycjEGDBsHW1hbdu3fHzJkzMWfOnEYIkRqTrh6JRdtWR9tU8tDlG/wwaELaUaTW/i7wcLSVOBq6nUwm4JNH28NWIcOO89ex9mi61CGRxOqdJB0+fBgDBgzQfb18+XK0aNECmzdvxhdffIE5c+Zg6dKljREjNaJk9kiyWu2DXGGrkCG7qFyXLFPj26ubauMokilq5uOECf2aAwCm/ZGIrMJSiSMiKdU7ScrOzkZQUJDu623btmHQoEG6r3v37o3Lly8bNThqfMnZRQBYtG2NlAo5OgW7AeCUW1PaW1W03aMZkyRT9VzPCEQFuiC/pALv/HqaI61WrN5JkoeHBzIyMgAAarUahw8fRmxsrO758vJy/odkhtgjybqxLqlppeUW40pOMeQyTUNPMk0KuQyzHu0AhUzA5n+uYcOpTKlDIonUO0nq1asXPvzwQ6SmpmLOnDlQq9Xo06eP7vnExESEhYU1RozUSMor1UjNLQEARLJHklXqwiSpSWmn2joEucLZji1TTFmbABe81FvTaPW99aeRW7UJOFmXeidJH330Ec6cOYOwsDC88cYbmDVrFhwdb32w/vDDD+jbt2+jBEmNI+VGMVRqEY62cng7K6UOhyTQOcQdcpmA9LwSpOUWSx2OxbPGpf/mbFzfZmju44TsonJM+4N7k1qjevdJCg8Px5kzZ5CYmAhvb28EBARUe/6DDz6oVrNEpk831ebtyP2SrJSjUoGoQFecSM3Docs3EOTuIHVIFksURV09UhzrkcyCUiHHrKHt8ej8vVh3LB2DOvijbytfqcOyGoWlFZKPuBrUGMfGxgYdOnSokSABQIcOHeDpyX/45oRF2wSwLqmpXLp+E9cKymCrkKFzCDcVNhedQtwx9p5wAMBba0+joLRC4oisQ+LVAtzzyTasOpQiaRzsHmjFWLRNANC1qoD4AJOkRrWvahQpJtQddjZyiaMhQ0y8vyXCPB2QWVCKmRvOSh2OxcsvqcCLy48gv6QCf57KhLq+G/81AiZJVizpOhtJEtAlzAOCoPnv4XphmdThWKw9FzX1SD2asR7J3NjbyvHxo+0BACsOpmDvxWyJI7Jcoihi8i8ncCWnGIFu9vhiREdJNxtmkmTFdN22ubLNqrk62KClrzMATfdtMj61WsS+JE2SxK1IzFO3CE883i0EAPDG2pMoLq+UOCLL9O3OJCQkXoOtXIZ5ozvDXeKu9EySrFRhaYVu1CCMI0lWryvrkhpVYkYB8ksq4KRUoH0gNxM2V2882AoBrnZIvVGCzzaflzoci7PvUg5mbdJMZ743uA06VDW7lZLBSVJYWBimTZuGlBRpi6no7lzO1iz39nJSwoX9Wqwek6TGpV3VFhvuwY2kzZiznQ1mPNIOALBkbzKOXOG/F2O5VlCKV1YchVoEHukciMe6hkgdEoAGJEmTJk3Cb7/9hoiICNx///1YuXIlyspYx2BukqpWtrEeiYBbxdtnMjUjHmRc2nqkONYjmb3eLX3waOcgiCLwf6tPorRCJXVIZq9Cpca45UeRXVSOVn7O+GhIO5NpS2NwkvTKK6/gyJEjOHLkCNq0aYNXX30V/v7+ePnll3H06NHGiJEaAVe20e18XOwQ7uUIUQT/Ojay8kq1rtaLm9pahnf+0xpeTkpcun4Tc/++IHU4Zu/jjWdx+EounJUKzH88Gva2prP6s8Hjvh06dMAXX3yB9PR0vPfee1i4cCG6dOmCDh06YPHixdzHzcQls2ib/qVLmKZ3D1sBGNfJtDwUl6vg4WirK5An8+bmYIvpQ6IAAN/sSMLp9HyJIzJff5y8ikW7kwEAnw3vYHJ/uDc4SaqoqMDPP/+MwYMHY9KkSYiJicHChQsxfPhwTJ06FaNHjzZmnGRk2uX/pvYfJEmna7hmlOMQkySj0k61dY/0lHQpMxnXg1F+GNjOHyq1iP9bfRIVKrXUIZmdi1mFeGP1SQDAC70i8UBbP4kjqqne25JoHT16FEuWLMGKFSsgl8vxxBNP4H//+x9atWqlOyc+Ph733nuvUQMl4xFFkSNJVIO28/bJtHyUlKtMasjbnOm2IuFUm8V5f3Bb7LmUjcSMAny74xJe7ttc6pDMxs2ySrzw41HcLFehW4QHJse3kDokvQweSerSpQsuXLiA+fPnIy0tDZ999lm1BAkA2rRpg5EjR9brevPmzUN4eDjs7OwQHR2NXbt21Xl+WVkZpk6ditDQUCiVSkRGRmLx4sW655cuXQpBEGo8SktL7+q+luR6URmKyiohE4BgD+7VRRpB7vbwd7VDpVrEsZRcqcOxCCXlKhxLyQMA9OCmthbH21mJ9we1BQB8ufUiLlwrlDgi8yCKIt5cewoXs4rg46zE3FGdTXbVp8FRJSUlYdOmTRg2bBhsbPQvHXd0dMSSJUvueK1Vq1ZhwoQJmDp1Ko4dO4aePXuif//+dbYXGD58OLZu3YpFixbh3LlzWLFiRY0kzcXFBRkZGdUednZ2d3VfS5JcNdUW5O4ApYKjBaQhCIKuFQDrkozj8JUbKFepEeBqh1BP/kFiiR7qGIC+rXxQrlLj9dUnoZJwCw1zsWzvZfx+4ioUMgHzRneGt7NS6pBqZXCSlJWVhQMHDtQ4fuDAARw+fNiga82ePRtjx47Fs88+i9atW2POnDkIDg7G/Pnz9Z6/adMm7NixAxs2bEC/fv0QFhaGrl27Ii4urtp5giDAz8+v2uNu7mtpuLKNasN+ScZ1+9J/U1nSTMYlCAI+ejgKzkoFjqfmYcmeZKlDMmlHruRi+p9nAABTBrRGTFX7EVNlcJI0btw4pKam1jienp6OcePG1fs65eXlOHLkCOLj46sdj4+Px969e/W+Zv369YiJicGsWbMQGBiIFi1aYPLkySgpKal2XlFREUJDQxEUFIT//Oc/OHbs2F3dF9BM8xUUFFR7mCvWI1FttP2SjqbkoryShah3ax/rkayCv6s93hrYGgDw2ZZzuFz1O5aqyy4qw7jlR1GpFjGwnT/G9AiTOqQ7MjhJSkxMROfOnWsc79SpExITE+t9nezsbKhUKvj6+lY77uvri8zMTL2vSUpKwu7du3H69GmsW7cOc+bMwerVq6slZ61atcLSpUuxfv16rFixAnZ2dujRowcuXLjQ4PsCwMyZM+Hq6qp7BAcH1/u9mppL3NiWatHMxwkejrYoq1TjFJc135X8kgrd9zCO9UgWb2SXYMRFeqK0Qo03156UdOd6U6RSi3h1xTFkFpQiwtsRnwxtbxajqwYnSUqlEteuXatxPCMjAwqFwYvlanyTRFGs9RunVqshCAKWL1+Orl27YsCAAZg9ezaWLl2qG03q1q0bHn/8cXTo0AE9e/bEzz//jBYtWmDu3LkNvi8ATJkyBfn5+bqHvtE0c5Fc1W073MtJ4kjI1AiCoOuXxCm3u3MgKQdqUTNi6+dqd+cXkFkTBAEfP9Ie9jZy7E+6gRWHrKPGtb5mJ5zD3ks5cLCV49vHo+GkNDxfkILBSdL999+vSxi08vLy8NZbb+H++++v93W8vLwgl8trjN5kZWXVGOXR8vf3R2BgIFxdb20Q2bp1a4iiiLS0NL2vkclkuhV5Db0voEkOXVxcqj3MUaVKjZQbmn3bwjndRnpo+yUdTM6ROBLztvdSVT0Sp9qsRoinA15/oCUAYOaGs7iaV3KHV1iHhMRr+HrbJQDAx4+2R3MzaqpqcJL0+eefIzU1FaGhoejTpw/69OmD8PBwZGZm4vPPP6/3dWxtbREdHY2EhIRqxxMSEmoUYmv16NEDV69eRVFRke7Y+fPnIZPJEBQUpPc1oiji+PHj8Pf3b/B9LUl6XgkqVCKUChn8XfjXLdWk7Zd0+HIuV+rcBW1/JC79ty5PxYWhc4gbisoq8da6U1a/+8SVnJuY+PNxAMDTcWEY3CFA2oAMZHCSFBgYiJMnT2LWrFlo06YNoqOj8cUXX+DUqVMG1+lMnDgRCxcuxOLFi3HmzBm89tprSElJwQsvvABAM8X15JNP6s5/7LHH4OnpiWeeeQaJiYnYuXMnXn/9dYwZMwb29vYAgA8++ACbN29GUlISjh8/jrFjx+L48eO6a9bnvpYs6baVbez+S/q09neBk1KBwrJKnMkw3wUKUrpeWIbz14ogCEC3CI4kWRO5TMCsoe1hK5dh+7nrWHcsXeqQJFNaocILPx5FYWklOoe44a0BraUOyWANmhR0dHTEc889d9c3HzFiBHJycjBt2jRkZGQgKioKGzZsQGhoKABNndPtvYucnJyQkJCAV155BTExMfD09MTw4cMxffp03Tl5eXl47rnnkJmZCVdXV3Tq1Ak7d+5E165d631fS6btkcSVbVQbuUxAdKg7dpy/jkOXbyAq0PXOL6JqtKNIbfxd4O5oK3E01NSa+ThjfL/m+HTzOXzweyLuae4FH2frGrkXRRFv/3oaZzIK4Oloi69Hd4atwjQbRtZFEBs4FpiYmIiUlBSUl5dXOz548GCjBGbqCgoK4Orqivz8fLOqT3r711P4cX8KxvWJxOsPtLrzC8gqfb3tIj7dfA79o/ww//FoqcMxO2+uOYmVh1Lx357hmDqwjdThkAQqVGoM+XoP/rlaYJX/jlYeTMGba09BJgA/jI1Fj2amM+1syOe3wSNJSUlJePjhh3Hq1CkIgqCbb9WuDFOpVA0ImZrKrUaSXNlGtYu9ranknVZ+Uk23irZN54OBmpaNXIZZQ9vjoa/2YOPpTGw4lYEB7fylDqtJnErLx7vr/wEATIpvaVIJkqEMHvsaP348wsPDce3aNTg4OOCff/7Bzp07ERMTg+3btzdCiGRM2uk2dtumurQLcoVSIUPOzXJdXy2qn9QbxUi5UQyFTECXcNPuJkyNq22AK17sHQkAePe308i9WX6HV5i/vOJyvLj8CMor1ejX2gcv9oqUOqS7YnCStG/fPkybNg3e3t6QyWSQyWS45557MHPmTLz66quNESMZSUm5ClfzNRv9spEk1UWpkKNTiBsA9ksy1L6qUaQOwW5m0wuGGs/LfZuhuY8TsovK8eEf9W+4bI7UahGvrTqOtNwShHg44PNhHc1+gZDBSZJKpYKTk2aqxsvLC1evXgUAhIaG4ty5c8aNjozqco5mRMDNwYbFpHRH2i1K2C/JMHt0S/+5qo00f3DMGtoeMgFYeywd285mSR1So/l620VsO3cdSoUM8x/vDFcHG6lDumsGJ0lRUVE4efIkACA2NhazZs3Cnj17MG3aNERERBg9QDKeJG5HQgbQNpU8dDlX4kjMhyiKunqk7qxHoiqdQtwxpkc4AOCtdadQWFohcUTGt+vCdcz+6zwAYPqQKLQNsIxVsQYnSW+//TbUas3Gl9OnT8eVK1fQs2dPbNiwAV9++aXRAyTj4XYkZIjOoW5QyASk55UgLbdY6nDMwsWsIlwvLINSIUPnUDepwyETMim+JUI9HZCRX4qZG89KHY5RpeeV4NUVxyCKwKiuwRgWY757m/6bwUnSAw88gEceeQQAEBERgcTERGRnZyMrKwt9+/Y1eoBkPNpGkuyRRPXhYKvQ9UhiXVL9aEeRuoR5QKmQSxwNmRJ7Wzk+fqQ9AOCnAym6XlrmrqxShZeWH0VucQWiAl3w3qC2UodkVAYlSZWVlVAoFDh9+nS14x4eHlwibAaSs7myjQxzeysAurM9FzUffHHNWI9ENXWP9MTo2BAAwJtrTqG4vFLiiO7e9D/O4ERqHlztbTB/dDTsbCzrjwODkiSFQoHQ0FD2QjJTTJLIUF2ZJNWbSi1ifxL7I1Hd3uzfCgGudki5UYzPt5yXOpy7su5YGn7YfwUAMGdERwR7OEgckfE1qCZpypQpuHGDvzTNSe7NcuQVa4oFwzyZJFH9xIR6QBA0U7XXC8ukDsekJV4tQEFpJZyVCkQFmE8XfmpaznY2+OiRdgCAxXuSceSKeS6MOJtZgClrTwEAXu3bDH1a+UgcUeMwOEn68ssvsWvXLgQEBKBly5bo3LlztQeZpqSqou1AN3vY21rWcCg1HlcHG7T0dQYAHLrMP4zqol36HxvhCYXc/PaooqbTp6UPHukcCFEE3lhzEmWV5jU7U1hagRd/PIrSCjV6NvfC+H4tpA6p0Rjc6WzIkCGNEAY1tiR22qYGig33wNnMQhxMvmE12yo0xK2tSFiPRHf27n/aYOf5bFzMKsLcrRcx+YGWUodUL6Io4vVfTiI5+yYCXO3wxchOkJt5w8i6GJwkvffee40RBzUy1iNRQ3UN98SyfVdwgHVJtSqvVONQ1ffHnPepoqbj5mCL6UPa4oUfj2L+jkt4MMpPt5rUlC3clYxN/2TCRi5g3uPR8LDwxsQcE7YSTJKoobqEuwPQ1CDkF1teEzxjOJ6ah5IKFbycbNHCl33IqH4ejPLHgHZ+UKlF/N/qk6hQqaUOqU4HknLw8SZNj6d3B7VFx2A3aQNqAgYnSTKZDHK5vNYHmSZdksQeSWQgH2c7RHg5QhSBw1c4mqSPdul/90gvtkMhg3wwOApuDjZIzCjAdzuTpA6nVlkFpXh5xTGo1CIe7hSIx6taGVg6g6fb1q1bV+3riooKHDt2DMuWLcMHH3xgtMDIeNRqUZckRbLbNjVAlzAPJGXfxMHLN3Bfa1+pwzE5+1iPRA3k7azEe4Pa4LVVJ/DFXxcQ38YXzasWS5iKCpUa4346iuuFZWjp64yPHo6ymj8GDE6SHnrooRrHhg4dirZt22LVqlUYO3asUQIj47maX4KySjVs5AIC3e2lDofMUNdwD6w6nMp+SXoUl1fiWKpmGTeTJGqIIR0Dsf74VWw7dx3/t+YkVr8QZ1LF0LM2ncWhy7lwUiow//HOcLA1OHUwW0arSYqNjcVff/1lrMuREWlHkUI9HU3qHx6ZD21TyVNp+RbRJdiYDl3ORYVKRKCbPUIssJkeNT5BEPDRw+3gpFTgWEoelu69LHVIOhtOZWDBrmQAwGfD2iPC27pmI4ySJJWUlGDu3LkICgoyxuXIyFi0TXcryN0eAa52qFSLOJaSJ3U4JkW7B1dcpKfVTEGQ8QW42eOtAa0BAJ9uPosrOTcljgi4dL0I/7f6JADguXsj8GCU9bUAMThJcnd3h4eHh+7h7u4OZ2dnLF68GJ9++mljxEh3SdsjKYJJEjWQIAi60SS2Aqhu70VNPRKX/tPdGtU1GN0jPFFaocaba05BFEXJYikur8SLPx5BUVkluoZ74P/MpI+TsRk8sfi///2v2l9LMpkM3t7eiI2Nhbu7u1GDI+PgSBIZQ5dwD/x6/KquHxAB+cUVOH01H4Bm81KiuyEIAj5+tB0emLMT+5JysOJgKh6TYBWZKIqYsvYUzl8rgrezEl+N6mS1XeQNTpKefvrpRgiDGpM2SbK2uWQyrtiqkaSjKbkor1TDVmGdvzRvty8pB6IINPNxgq+LndThkAUI9XTE6w+0wod/JGLGhjPo3dIbAW5Nu+Dmx/1X8Nvxq5DLBHz9WGf4WPF/2wb/lluyZAl++eWXGsd/+eUXLFu2zChBkfGUVaqQllsMgCNJdHcivZ3g4WiLsko1TqXnSR2OSdh3Wz0SkbE8HReGziFuKCqrxNR1TTvtdiwlF9P+SAQATOnfSjfNbq0MTpI+/vhjeHnVnHv38fHBjBkzjBIUGU9KTjHUIuCsVMDLybLbx1PjEgQBXcNYl3S7PeyPRI1ALhMwa2h72Mpl2HbuOtYdS2+S++YUleGl5UdRoRLRP8oPY+8Jb5L7mjKDk6QrV64gPLzmNy40NBQpKSlGCYqMJ+m2TttceUN3S/tXJfslaToQX8wqgiAA3SKYJJFxNfNxxvh+zQEA0/5IxPXCska9n0otYvzK48jIL0WElyNmDW3Pzww0IEny8fHByZMnaxw/ceIEPD35i8LUsGibjEmbJB25nAuVWrqVN6ZgX5JmFKltgAvcHDhKS8b33L0RaOPvgrziCry3/nSj3mvOX+ex+2I27G3kmP94NJztbBr1fubC4CRp5MiRePXVV7Ft2zaoVCqoVCr8/fffGD9+PEaOHNkYMdJdSL7OJImMp7W/C5yVChSWVeJMRoHU4UhKu19bj0gu/afGYSOXYdbQ9pDLBGw4lYmNpzIa5T5/n72GuX9fBADMfKQdWvqZ1rYoUjI4SZo+fTpiY2Nx3333wd7eHvb29oiPj0ffvn1Zk2SCuLKNjEkuExAdpmn1Ye1Tbnur6pG49J8aU1SgK17sFQkAeOe3f5BXXG7U66feKMaElccBAE92D8WQToFGvb65MzhJsrW1xapVq3Du3DksX74ca9euxaVLl7B48WLY2nLI2dQkZRcBYCNJMh7WJWkWRKTllkAhE6x+9Q81vlfua4ZmPk7ILirTrTwzhtIKFV748QgKSivRMdgNUwe2Ntq1LUWDd6lr3rw5mjdvbsxYyMjySyqQXaT5qyOMSRIZibZf0sHLNyCKolUWd2q3IukU4mZVm32SNJQKOT55tD2GfrMXa4+mY1D7APRp5XPX131//T/452oBPBxtMW90ZygVciNEa1kMHkkaOnQoPv744xrHP/30UwwbNswoQZFxXK6aavNxVsJJyV/kZBztAt2gVMhw42Y5Ll0vkjocSezRTbWxHomaRnSoO8b00Kwsf2vdKRSWVtzV9X4+lIqVh1IhCMCXIzs1ecNKc2FwkrRjxw4MHDiwxvEHH3wQO3fuNEpQZBxc2UaNwVYhQ+cQbV1SrsTRND1RFHVNJHuwHoma0OT4lgjxcEBGfik+3ni2wdc5nZ6Pd37TrJabdH8L3NOcyX5tDE6SioqK9NYe2djYoKDAule7mJokXdE2kyQyri66uqQciSNpeheyipBdVA47Gxk6hrhJHQ5ZEXtbOT5+tB0AYPmBFOy7ZPi/v/ziCry0/CjKKtXo28oHL/VuZuwwLYrBSVJUVBRWrVpV4/jKlSvRpk0bowRFxqFb2ebFlW1kXNq6pAPJNyTdqVwK2qX/XcI8WMNBTS4u0ku36e2ba0+ipFxV79eq1SIm/nwcKTeKEeRuj/8N7wiZzPpqCg1hcKHKO++8g0cffRSXLl1C3759AQBbt27FihUr9O7pRtJJqqoX4XQbGVunEDcoZAIy8kuRlluCYA8HqUNqMnt1W5FwioKkMaV/K2w7m4UrOcX4fMs5vP2f+g1QzN9xCVvPZsFWIcM3j0fD1YENI+/E4JGkwYMH49dff8XFixfx0ksvYdKkSUhLS8Nff/2FIUOGNEKI1BCiKN6qSeJ0GxmZg60C7YJcAVhXK4BKlRr7qzpt92jGeiSShrOdDWY8rJl2W7QnGUdT7lwbuPtCNj7fcg4A8OFDbREV6NqoMVoKg5MkABg4cCD27NmDmzdvIjs7G3///Td69eqF48ePG3ytefPmITw8HHZ2doiOjsauXbvqPL+srAxTp05FaGgolEolIiMjsXjxYt3zCxYsQM+ePeHu7g53d3f069cPBw8erHaN999/H4IgVHv4+fkZHLspyyosQ3G5CnKZgGB36/krn5qONfZL+udqAQpLK+Fip0DbAH7IkHT6tPLBI50CIYrA/60+ibLK2qfdMvJL8OrKY1CLwPCYIIzoEtKEkZq3BiVJt8vPz8e8efPQuXNnREdHG/TaVatWYcKECZg6dSqOHTuGnj17on///nVulDt8+HBs3boVixYtwrlz57BixQq0atVK9/z27dsxatQobNu2Dfv27UNISAji4+ORnl59F+W2bdsiIyND9zh16pRhb9zEJVVtRxLsbg9bxV3/mIlq6BqmSZIOXbaeJGlP1aq2bhGekLOWgyT2zn/awMvJFhezivBV1bYi/1ZeqcZLy4/ixs1ytA1wwbSHopo4SvPW4E/Pv//+G6NHj4a/vz/mzp2LAQMG4PDhwwZdY/bs2Rg7diyeffZZtG7dGnPmzEFwcDDmz5+v9/xNmzZhx44d2LBhA/r164ewsDB07doVcXFxunOWL1+Ol156CR07dkSrVq2wYMECqNVqbN26tdq1FAoF/Pz8dA9vb2/DvwkmjMv/qbHFhHpAEDSrKLMKS6UOp0ns09UjcaqNpOfuaIsPq5Keedsv4Z+r+TXOmbHhDI6l5MHFToH5o6NhZ8PFBoYwKElKS0vD9OnTERERgVGjRsHDwwMVFRVYs2YNpk+fjk6dOtX7WuXl5Thy5Aji4+OrHY+Pj8fevXv1vmb9+vWIiYnBrFmzEBgYiBYtWmDy5MkoKSmp9T7FxcWoqKiAh0f1rQMuXLiAgIAAhIeHY+TIkUhKSqoz3rKyMhQUFFR7mDJt0Tb3bKPG4upgg1Z+LgCAQ1bQL6msUqUbNYtrxqJtMg392/mjf5QfVGoR/7f6JCpUat1zvx1Px9K9lwEA/xvRESGeLL0wVL2TpAEDBqBNmzZITEzE3LlzcfXqVcydO7fBN87OzoZKpYKvr2+1476+vsjMzNT7mqSkJOzevRunT5/GunXrMGfOHKxevRrjxo2r9T5vvvkmAgMD0a9fP92x2NhYfP/999i8eTMWLFiAzMxMxMXFISen9p4TM2fOhKurq+4RHBxs4DtuWhxJoqYQa0X9ko6l5KG0Qg0vJyWa+/CPDzIdHzzUFq72NvjnagG+26n5g//8tUK8uUZTRvJyn2a4r7VvXZegWtQ7SdqyZQueffZZfPDBBxg4cCDkcuMM2f1736e69oJSq9UQBAHLly9H165dMWDAAMyePRtLly7VO5o0a9YsrFixAmvXroWdnZ3ueP/+/fHoo4+iXbt26NevH/78808AwLJly2qNc8qUKcjPz9c9UlNTG/J2m8ytHklMkqjxdL2tX5Kl23vbVJs17ldHpsvH2Q7vDdK0Afjirws4npqHF348gpIKFXo088Rr97eQOELzVe8kadeuXSgsLERMTAxiY2Px1Vdf4fr16w2+sZeXF+RyeY1Ro6ysrBqjS1r+/v4IDAyEq+utVSWtW7eGKIpIS0urdu5nn32GGTNmYMuWLWjfvn2dsTg6OqJdu3a4cOFCrecolUq4uLhUe5iqCpUaKTeKAXD5PzWuLlXF2+euFSK/+O72kjJ1e6uaSHLpP5mihzsFondLb5Sr1Bj2zV4kXb8Jf1c7fDmyExcZ3IV6J0ndu3fHggULkJGRgeeffx4rV65EYGAg1Go1EhISUFhYaNCNbW1tER0djYSEhGrHExISqhVi365Hjx64evUqiopubap5/vx5yGQyBAUF6Y59+umn+PDDD7Fp0ybExMTcMZaysjKcOXMG/v7+Br0HU5WWW4JKtQh7Gzl8ne3u/AKiBvJ2ViLCyxGiCBy+YrmjSTfLKnE8NQ8Am0iSaRIEATMebgcnpQIVKhE2cgFfj+4MTyel1KGZNYNXtzk4OGDMmDHYvXs3Tp06hUmTJuHjjz+Gj48PBg8ebNC1Jk6ciIULF2Lx4sU4c+YMXnvtNaSkpOCFF14AoJnievLJJ3XnP/bYY/D09MQzzzyDxMRE7Ny5E6+//jrGjBkDe3vNDsazZs3C22+/jcWLFyMsLAyZmZnIzMysllhNnjwZO3bsQHJyMg4cOIChQ4eioKAATz31lKHfDpOUnH2r0zZbzlNjs4Z+SQcv30ClWkSwh71VdRcn8xLgZo8Ph7SFo60cHz4UpduImhrurhrotGzZErNmzUJaWhpWrFhh8OtHjBiBOXPmYNq0aejYsSN27tyJDRs2IDQ0FACQkZFRrWeSk5MTEhISkJeXh5iYGIwePRqDBg3Cl19+qTtn3rx5KC8vx9ChQ+Hv7697fPbZZ7pz0tLSMGrUKLRs2RKPPPIIbG1tsX//ft19zZ22RxKn2qgpWENdkm7pfwRHkci0PdwpCP9MexAju7JhpDEIorXtTmkkBQUFcHV1RX5+vsnVJ7217hR+OpCCV/o2w6T4llKHQxYuLbcY93yyDQqZgBPvxcNRafCWkCZv4Je78M/VAnwxsiMe6hgodThEdBcM+fxmK2YLlHydy/+p6QS5OyDQzR6VahHHUvKkDsfocm+WIzFD0xetO5tIElkVJkkWiD2SqKnp6pIscIuSA8k5EEWguY8TfLgQgsiqMEmyMDfLKpFZoNkiIsKLDe+oaWhbAVhiU8k9FzXvqQe7bBNZHSZJFuZyjmYUydPRFq4ONhJHQ9ZCO5J0LCWvzt3IzdHeqk1tOdVGZH2YJFmYJNYjkQQivR3h6WiLsko1TqXV3GTTXGXml+LS9ZuQCUC3CCZJRNaGSZKFYT0SSUEQBItsBbAvSTOKFBXoCld7jswSWRsmSRZGlySxRxI1MW2SdMiCire19UicaiOyTkySLEwSN7YliWiLtw9fzoVKbf7t10RR1DWR7MGtSIisEpMkCyKKIpKva7ck4co2alqt/V3grFSgqKwSZ6r6CpmzlBvFSM8rgY1cQEwYt3cgskZMkizIjZvlKCithCAAoZ7cX4qallx2K5mwhLok7VRbpxB3ONhaXhdxIrozJkkWRDvVFuhmDzsbucTRkDXqGq6p3bGEfknapf9xrEcislpMkiwItyMhqd0q3s6FOW8LqVbfVo/EJpJEVotJkgVh0TZJrV2gK+xsZLhxsxyXqurjzNH5rELk3CyHvY0cHYLcpA6HiCTCJMmCJGdri7aZJJE0bBUydAo2/7okbT1Sl3AP2Cr4a5LIWvFfvwW51SOJK9tIOrrNbs04SdpXVY/Ug/VIRFaNSZKFUKlFXM4pBsDpNpJWrLbzdtINs6xLqlSpcSBJk+DFsT8SkVVjkmQhruaVoLxSDVuFDAFu9lKHQ1asU4g7FDIBmQWlSMstkTocg51Kz0dhWSVc7W3QJsBF6nCISEJMkiyEtmg7zNMBcpkgcTRkzext5Wgf5ArAPKfc9latausW4cF/S0RWjkmShbjVaZtTbSS9W/2SzDFJqqpH4tJ/IqvHJMlC6Iq2uR0JmYCu4ZoVbgfNbLPb0goVDl/OBcAmkkTEJMlisEcSmZLoUA8IgiZ5zyoolTqcejuakouySjV8nJWI5CpRIqvHJMlCaEeSIryZJJH0XO1t0NpPU/RsTqNJ2i7bcZGeEATWIxFZOyZJFqC0QoX0PM0qItYkkakwx35Jey5q92tjPRIRMUmyCFdyiiGKgIudAh6OtlKHQwTgVr8kc0mSisoqcSItHwAQ14z1SETEJMki6LYj8XbiFAGZjJgwTZJ07loh8orLJY7mzg4l34BKLSLEwwFB7g5Sh0NEJoBJkgVg0TaZIm9nJSK8HSGK0K0YM2XaqbYeHEUioipMkixA8nUmSWSadFNuZlC8rW0i2Z31SERUhUmSBUjSbWzLJIlMi7Z4+4CJ1yXduFmOxIwCAED3CI4kEZEGkyQLcKuRJJMkMi3aztv/pOfjZlmlxNHUbn+SZhSppa8zvJ2VEkdDRKaCSZKZyysux42bmqLYME8mSWRaAt3sEehmj0q1iGMpeVKHUyttPVJ3dtkmotswSTJz2lEkPxc7OCoVEkdDVNOtfkk5EkdSO20TSe7XRkS3Y5Jk5jjVRqbO1OuSMvJLkJR9EzLhVqxERACTJLPH7UjI1GkTj2OpeSirVEkcTU17L2pGkdoFucHV3kbiaIjIlDBJMnNJ1zmSRKYtwssRXk62KK9U42RVR2tTsve2/dqIiG7HJMnMJXEkiUycIAgmu4+bKIrYe6mqiST7IxHRvzBJMmNqtYjLupokJ4mjIapd1zDTTJIu5xQjI78UtnIZokPdpQ6HiEyM5EnSvHnzEB4eDjs7O0RHR2PXrl11nl9WVoapU6ciNDQUSqUSkZGRWLx4cbVz1qxZgzZt2kCpVKJNmzZYt27dXd/XFF0rLEVJhQoKmYAgd3upwyGqVZeqkaQjV3JRqVJLHM0t2qX/nULcYG8rlzgaIjI1kiZJq1atwoQJEzB16lQcO3YMPXv2RP/+/ZGSklLra4YPH46tW7di0aJFOHfuHFasWIFWrVrpnt+3bx9GjBiBJ554AidOnMATTzyB4cOH48CBA3d1X1Ok3Y4kxMMBNnLJ812iWrXyc4GznQJFZZU4k1EodTg6XPpPRHURRFEUpbp5bGwsOnfujPnz5+uOtW7dGkOGDMHMmTNrnL9p0yaMHDkSSUlJ8PDQv1R3xIgRKCgowMaNG3XHHnzwQbi7u2PFihUNuq8+BQUFcHV1RX5+PlxcXOr1GmP7cf8VvP3rafRr7YOFT3WRJAai+hqz9BD+PpuFtwe2xrM9I6QOB2q1iOjpCcgtrsDqF7ojJozL/4msgSGf35INP5SXl+PIkSOIj4+vdjw+Ph579+7V+5r169cjJiYGs2bNQmBgIFq0aIHJkyejpKREd86+fftqXPOBBx7QXbMh9wU003wFBQXVHlLjyjYyJ9ri7UMmstnt2cxC5BZXwMFWjg7BblKHQ0QmSLIWzdnZ2VCpVPD19a123NfXF5mZmXpfk5SUhN27d8POzg7r1q1DdnY2XnrpJdy4cUNXl5SZmVnnNRtyXwCYOXMmPvjgA4PfZ2NKzi4CwKJtMg+3r3ATRRGCIEgaj3ZVW9dwD05XE5Fekv9m+Pcvyrp+earVagiCgOXLl6Nr164YMGAAZs+ejaVLl1YbTarPNQ25LwBMmTIF+fn5ukdqamq93l9jYrdtMidRAa6ws5Eht7gCF7OKpA5H1x+JS/+JqDaSJUleXl6Qy+U1Rm+ysrJqjPJo+fv7IzAwEK6urrpjrVu3hiiKSEtLAwD4+fnVec2G3BcAlEolXFxcqj2kVF6pRmquJjFkjyQyB7YKGTqHaJbZS71FSYVKjQNJmiSJm9oSUW0kS5JsbW0RHR2NhISEascTEhIQFxen9zU9evTA1atXUVR066/Q8+fPQyaTISgoCADQvXv3GtfcsmWL7poNua8pSs0thkotwtFWDh9npdThENWLqTSVPJmWj5vlKrg52KCNv7R/8BCR6ZJ0um3ixIlYuHAhFi9ejDNnzuC1115DSkoKXnjhBQCaKa4nn3xSd/5jjz0GT09PPPPMM0hMTMTOnTvx+uuvY8yYMbC31/QJGj9+PLZs2YJPPvkEZ8+exSeffIK//voLEyZMqPd9zYF2+X+4t6PktR1E9fXvuiSp7KuqR+oe4QmZjP9+iEg/yQq3Ac1y/ZycHEybNg0ZGRmIiorChg0bEBoaCgDIyMio1rvIyckJCQkJeOWVVxATEwNPT08MHz4c06dP150TFxeHlStX4u2338Y777yDyMhIrFq1CrGxsfW+rzlIYtE2maFOwe6wkQvILChFWm4Jgj0cJIljz0Xu10ZEdyZpnyRzJnWfpClrT2LFwVS8el9zTLy/RZPfn6ihHp2/F0eu5OKzYR0wNDqoye9fWqFC+w+2oLxSja2TeiHSm39oEFkTs+iTRHdH2yMpgivbyMx00e3jliPJ/Y9eyUV5pRq+Lkr++yGiOjFJMlNc/k/mKlbi4u09VfVIPSK9WM9HRHVikmSGisoqkVVYBkBTuE1kTqLD3CEIwOWcYmQVlDb5/bX9kbj0n4juhEmSGbpcNYrk5aSEi52NxNEQGcbF7tay+4NNvEVJYWkFTqblAwDiuKktEd0BkyQzdOm6ZmUb6ynIXEnVL+lg8g2o1CLCPB0Q6GbfpPcmIvPDJMkMsR6JzJ1UdUnapf/duRUJEdUDkyQzpEuSWI9EZiqmaoXb2cxC5BWXN9l9tZva9mjGeiQiujMmSWaII0lk7ryclIisSvIPXc5tkntmF5XhbGYhAE2nbSKiO2GSZGZEUdRtScKaJDJnXcM1iUpT9UvaX7WhbSs/Z3g6cb9DIrozJklmJruoHIVllZAJQIinNFs6EBmDri6piUaStEv/41iPRET1xCTJzCRVrWwLcneAUiGXOBqihtOucDudno+bZZWNfr+9F1mPRESGYZJkZliPRJYiwM0egW72UKlFHE1p3NGk9LwSXM4phlwm6JIzIqI7YZJkZpgkkSVpqlYA2lGkdoGucGYDViKqJyZJZiapKkmK4PJ/sgDaUZ0DjZwk7auqR+JUGxEZgkmSmdGOJEV4OUkcCdHd0yZJx1PzUFapapR7iKKo29SWRdtEZAgmSWakUqXGlRw2kiTLEe7lCC8nJcor1bo91YwtKfsmrhWUwVYhQ3Soe6Pcg4gsE5MkM5KeV4IKlQilQgZ/FzupwyG6a4IgNHpdknbpf3SIO+xsuCKUiOqPSZIZSbqtaFsmEySOhsg4uoRpRncaqy6JS/+JqKGYJJkRbadtrmwjS6LtvH3k8g1UqtRGvbZaLWJfEje1JaKGYZJkRrj8nyxRSz9nuNgpcLNchcSMAqNeOzGjAHnFFXC0laN9kKtRr01Elo9JkhnRrWzz5so2shxymYAuYY1Tl6Rd+h8b4QkbOX/dEZFh+FvDjGi3JOFIElmaro1UvH1r6T/rkYjIcEySzERJuQpX80sBABFMksjCaJOkQ5dvQK0WjXLNCpVal3SxPxIRNQSTJDNxuao/kpuDDdwdbSWOhsi4ogJdYW8jR25xBS5WjZjerROpeSguV8HD0Rat/JyNck0isi5MkswEi7bJktnIZegc6gbAeK0AtP2Rukd4smUGETUIkyQzwSSJLF3XME3d0CGjJUmaeqTurEciogZikmQmkqp6JEVyZRtZqNuLt0Xx7uqSSspVOHolDwCLtomo4ZgkmYmkbK5sI8vWKcQNNnIBmQWlSL1RclfXOnIlF+UqNfxd7fhvhogajEmSmeB0G1k6Oxs52ge5AQAOJOfc1bX23DbVJgisRyKihmGSZAZyb5Yjr7gCABDmySSJLJex+iVpi7Z7cOk/Ed0FJklmQLuxbYCrHextuYs5WS5dknS54UlSfkkFTqXlAQDiuKktEd0FJklmQDfV5s1RJLJs0aHukAnAlZxiXCsobdA1DibfgFrUNF31d7U3coREZE2YJJmB5Kqi7Qgvrmwjy+ZiZ4M2AS4AGj7lxqX/RGQsTJLMgHb5P4u2yRpo+yU1OEm6qKlH4lYkRHS3mCSZAU63kTXpGu4OoGFJ0vXCMpy7VgiAI0lEdPeYJJk4tVrUJUnc2JasQZcwTfH2uWuFyL1ZbtBr9yVpRpFa+7vAg3scEtFdkjxJmjdvHsLDw2FnZ4fo6Gjs2rWr1nO3b98OQRBqPM6ePas7p3fv3nrPGThwoO6c999/v8bzfn5+jfo+GyqjoBRllWrYyAUEurEIlSyfp5MSzXw09XeHr+Qa9Np9VfVIPTiKRERGoJDy5qtWrcKECRMwb9489OjRA99++y369++PxMREhISE1Pq6c+fOwcXFRfe1t7e37v+vXbsW5eW3/vrMyclBhw4dMGzYsGrXaNu2Lf766y/d13K5aS6tT66qRwrxcIBCLnlOS9QkuoZ74GJWEQ4m5+D+Nr71ft0ebT0Sl/4TkRFImiTNnj0bY8eOxbPPPgsAmDNnDjZv3oz58+dj5syZtb7Ox8cHbm5uep/z8PCo9vXKlSvh4OBQI0lSKBQmO3p0O93KNu7ZRlYkNtwDPx1IMaguKfVGMVJuFEMuE9A1nEkSEd09yYYmysvLceTIEcTHx1c7Hh8fj71799b52k6dOsHf3x/33Xcftm3bVue5ixYtwsiRI+HoWL2e58KFCwgICEB4eDhGjhyJpKSkOq9TVlaGgoKCao+mcOk665HI+mjrkk5fLUBRWWW9XqOtR+oQ5AonpaR//xGRhZAsScrOzoZKpYKvb/WhdF9fX2RmZup9jb+/P7777jusWbMGa9euRcuWLXHfffdh586des8/ePAgTp8+rRup0oqNjcX333+PzZs3Y8GCBcjMzERcXBxycmrfL2rmzJlwdXXVPYKDgw18xw3DPdvIGgW42SPI3R4qtYij9axL2ntRU4/Epf9EZCyS/7n1780nRVGsdUPKli1bomXLlrqvu3fvjtTUVHz22We49957a5y/aNEiREVFoWvXrtWO9+/fX/f/27Vrh+7duyMyMhLLli3DxIkT9d57ypQp1Z4rKChokkSJSRJZq67hHkjLTcfB5Bu4t4V3neeKoqjbr431SERkLJKNJHl5eUEul9cYNcrKyqoxulSXbt264cKFCzWOFxcXY+XKlTVGkfRxdHREu3bt9F5HS6lUwsXFpdqjsZVVqpCWWwyAPZLI+sQasI/bpetFyCosg1IhQ+cQ98YOjYishGRJkq2tLaKjo5GQkFDteEJCAuLi4up9nWPHjsHf37/G8Z9//hllZWV4/PHH73iNsrIynDlzRu91pJR6oxhqEXBWKuDtpJQ6HKImpS2+Pp6ah9IKVZ3nakeRYsLcYWdjmitVicj8SDrdNnHiRDzxxBOIiYlB9+7d8d133yElJQUvvPACAM0UV3p6Or7//nsAmtVvYWFhaNu2LcrLy/Hjjz9izZo1WLNmTY1rL1q0CEOGDIGnZ82h98mTJ2PQoEEICQlBVlYWpk+fjoKCAjz11FON+4YNpC3aDvd2rHUKkshShXk6wNtZieuFZTiZlo+u4R61nruH9UhE1AgkTZJGjBiBnJwcTJs2DRkZGYiKisKGDRsQGhoKAMjIyEBKSoru/PLyckyePBnp6emwt7dH27Zt8eeff2LAgAHVrnv+/Hns3r0bW7Zs0XvftLQ0jBo1CtnZ2fD29ka3bt2wf/9+3X1NBeuRyJoJgoCuYR7481QGDibn1JokqdQi9idppuTi2ESSiIxIEEVRlDoIc1RQUABXV1fk5+c3Wn3SG6tPYtXhVEzo1xwT+rVolHsQmbJley/jvfX/oGdzL/wwNlbvOafS8jHoq91wVipw7N372XSViOpkyOc3f5uYMI4kkbXTjh4dvZKLSpVa7zl7q7YiiY3wYIJEREbF3ygmLEm3sS27bZN1aunrDBc7BW6Wq5CYob+Bq7ZouzvrkYjIyJgkmaiC0gpkF5UB4PJ/sl4ymaAbTdK3RUl5pVp3nPVIRGRsTJJMlHZjWx9nJbdYIKumTZIO6EmSTqTloaRCBU9HW7T0dW7q0IjIwjFJMlGsRyLS0O7jdujyDajV1deZaJf+d4v0hEzGNhlEZFxMkkyUrh6JU21k5aICXWFvI0decQUuZBVVe05bj9SD9UhE1AiYJJkojiQRadjIZYgO1Ww1cvsWJcXllTiWotn8lvVIRNQYmCSZqORszV/M4VzZRqS3ePvw5VxUqEQEutkj1NNBqtCIyIIxSTJBoijqCrc53UZ0e5KUA23/21tL/z25bQ8RNQomSSYoq7AMN8tVkMsEBLvzL2SijsFusJXLcK2gDCk3igHcaiLJqTYiaixMkkxQUtUoUrC7PWwV/BER2dnI0T7IFYCmFUB+cQVOp+cD4Ka2RNR4+Alsgli0TVTT7XVJ+5NzoBY109F+rnYSR0ZElopJkgli0TZRTdok6dDlG9jHpf9E1ASYJJkg3UgSi7aJdKJD3SETgCs5xfjzVAYA1iMRUeNikmSCtI0kIzndRqTjbGeDtgGauqTrhWUQBKBbBJMkImo8TJJMTIVKjZQczeodjiQRVafdogQA2vi7wN3RVsJoiMjSMUkyMWm5JahUi7C3kcPXmQWpRLfT1iUBnGojosbHJMnEaIu2w7wcuWEn0b90CXPX/f+4ZizaJqLGpZA6AKouv6QCTkoFIliPRFSDp5MSj8WG4HL2TXRnPRIRNTJB1Pb4J4MUFBTA1dUV+fn5cHFxMeq1RVFEaYUa9rZyo16XiIjI2hny+c3pNhMkCAITJCIiIokxSSIiIiLSg0kSERERkR5MkoiIiIj0YJJEREREpAeTJCIiIiI9mCQRERER6cEkiYiIiEgPJklEREREejBJIiIiItKDSRIRERGRHkySiIiIiPRgkkRERESkB5MkIiIiIj0UUgdgrkRRBAAUFBRIHAkRERHVl/ZzW/s5XhcmSQ1UWFgIAAgODpY4EiIiIjJUYWEhXF1d6zxHEOuTSlENarUaV69ehbOzMwRBkDock1RQUIDg4GCkpqbCxcVF6nCsHn8epoU/D9PCn4fpaayfiSiKKCwsREBAAGSyuquOOJLUQDKZDEFBQVKHYRZcXFz4S8eE8OdhWvjzMC38eZiexviZ3GkESYuF20RERER6MEkiIiIi0oNJEjUapVKJ9957D0qlUupQCPx5mBr+PEwLfx6mxxR+JizcJiIiItKDI0lEREREejBJIiIiItKDSRIRERGRHkySiIiIiPRgkkRGNXPmTHTp0gXOzs7w8fHBkCFDcO7cOanDoiozZ86EIAiYMGGC1KFYtfT0dDz++OPw9PSEg4MDOnbsiCNHjkgdllWqrKzE22+/jfDwcNjb2yMiIgLTpk2DWq2WOjSrsHPnTgwaNAgBAQEQBAG//vprtedFUcT777+PgIAA2Nvbo3fv3vjnn3+aLD4mSWRUO3bswLhx47B//34kJCSgsrIS8fHxuHnzptShWb1Dhw7hu+++Q/v27aUOxarl5uaiR48esLGxwcaNG5GYmIjPP/8cbm5uUodmlT755BN88803+Oqrr3DmzBnMmjULn376KebOnSt1aFbh5s2b6NChA7766iu9z8+aNQuzZ8/GV199hUOHDsHPzw/333+/bv/UxsYWANSorl+/Dh8fH+zYsQP33nuv1OFYraKiInTu3Bnz5s3D9OnT0bFjR8yZM0fqsKzSm2++iT179mDXrl1Sh0IA/vOf/8DX1xeLFi3SHXv00Ufh4OCAH374QcLIrI8gCFi3bh2GDBkCQDOKFBAQgAkTJuCNN94AAJSVlcHX1xeffPIJnn/++UaPiSNJ1Kjy8/MBAB4eHhJHYt3GjRuHgQMHol+/flKHYvXWr1+PmJgYDBs2DD4+PujUqRMWLFggdVhW65577sHWrVtx/vx5AMCJEyewe/duDBgwQOLIKDk5GZmZmYiPj9cdUyqV6NWrF/bu3dskMXCDW2o0oihi4sSJuOeeexAVFSV1OFZr5cqVOHr0KA4dOiR1KAQgKSkJ8+fPx8SJE/HWW2/h4MGDePXVV6FUKvHkk09KHZ7VeeONN5Cfn49WrVpBLpdDpVLho48+wqhRo6QOzeplZmYCAHx9fasd9/X1xZUrV5okBiZJ1GhefvllnDx5Ert375Y6FKuVmpqK8ePHY8uWLbCzs5M6HAKgVqsRExODGTNmAAA6deqEf/75B/Pnz2eSJIFVq1bhxx9/xE8//YS2bdvi+PHjmDBhAgICAvDUU09JHR5BMw13O1EUaxxrLEySqFG88sorWL9+PXbu3ImgoCCpw7FaR44cQVZWFqKjo3XHVCoVdu7cia+++gplZWWQy+USRmh9/P390aZNm2rHWrdujTVr1kgUkXV7/fXX8eabb2LkyJEAgHbt2uHKlSuYOXMmkySJ+fn5AdCMKPn7++uOZ2Vl1RhdaiysSSKjEkURL7/8MtauXYu///4b4eHhUodk1e677z6cOnUKx48f1z1iYmIwevRoHD9+nAmSBHr06FGjLcb58+cRGhoqUUTWrbi4GDJZ9Y9CuVzOFgAmIDw8HH5+fkhISNAdKy8vx44dOxAXF9ckMXAkiYxq3Lhx+Omnn/Dbb7/B2dlZN6fs6uoKe3t7iaOzPs7OzjXqwRwdHeHp6ck6MYm89tpriIuLw4wZMzB8+HAcPHgQ3333Hb777jupQ7NKgwYNwkcffYSQkBC0bdsWx44dw+zZszFmzBipQ7MKRUVFuHjxou7r5ORkHD9+HB4eHggJCcGECRMwY8YMNG/eHM2bN8eMGTPg4OCAxx57rGkCFImMCIDex5IlS6QOjar06tVLHD9+vNRhWLXff/9djIqKEpVKpdiqVSvxu+++kzokq1VQUCCOHz9eDAkJEe3s7MSIiAhx6tSpYllZmdShWYVt27bp/cx46qmnRFEURbVaLb733nuin5+fqFQqxXvvvVc8depUk8XHPklEREREerAmiYiIiEgPJklEREREejBJIiIiItKDSRIRERGRHkySiIiIiPRgkkRERESkB5MkIiIiIj2YJBER1UPv3r0xYcIEqcMgoibEJImIzNLTTz+NIUOGVDu2evVq2NnZYdasWdIERUQWhXu3EZFFWLhwIcaNG4evv/4azz77rNThEJEF4EgSEZm9WbNm4eWXX8ZPP/1UI0HavHkz7OzskJeXV+34q6++il69egEAcnJyMGrUKAQFBcHBwQHt2rXDihUr6rynIAj49ddfqx1zc3PD0qVLdV+np6djxIgRcHd3h6enJx566CFcvny5oW+TiJoYkyQiMmtvvvkmPvzwQ/zxxx949NFHazzfr18/uLm5Yc2aNbpjKpUKP//8M0aPHg0AKC0tRXR0NP744w+cPn0azz33HJ544gkcOHCgwXEVFxejT58+cHJyws6dO7F79244OTnhwQcfRHl5eYOvS0RNh9NtRGS2Nm7ciN9++w1bt25F37599Z4jl8sxYsQI/PTTTxg7diwAYOvWrcjNzcWwYcMAAIGBgZg8ebLuNa+88go2bdqEX375BbGxsQ2KbeXKlZDJZFi4cCEEQQAALFmyBG5ubti+fTvi4+MbdF0iajocSSIis9W+fXuEhYXh3XffRWFhIQCgf//+cHJygpOTE9q2bQsAGD16NLZv346rV68CAJYvX44BAwbA3d0dgGZk6aOPPkL79u3h6ekJJycnbNmyBSkpKQ2O7ciRI7h48SKcnZ118Xh4eKC0tBSXLl26y3dORE2BI0lEZLYCAwOxZs0a9OnTBw8++CA2bdqEhQsXoqSkBABgY2MDAOjatSsiIyOxcuVKvPjii1i3bh2WLFmiu87nn3+O//3vf5gzZw7atWsHR0dHTJgwoc5pMUEQIIpitWMVFRW6/69WqxEdHY3ly5fXeK23t/ddvW8iahpMkojIrIWEhGDHjh3o06cP4uPjsXnzZgQGBtY477HHHsPy5csRFBQEmUyGgQMH6p7btWsXHnroITz++OMANAnOhQsX0Lp161rv6+3tjYyMDN3XFy5cQHFxse7rzp07Y9WqVfDx8YGLi4sx3ioRNTFOtxGR2QsKCsL27duRk5OD+Ph45Ofn1zhn9OjROHr0KD766CMMHToUdnZ2uueaNWuGhIQE7N27F2fOnMHzzz+PzMzMOu/Zt29ffPXVVzh69CgOHz6MF154QTdypb2fl5cXHnroIezatQvJycnYsWMHxo8fj7S0NOO9eSJqNEySiMgiBAYGYseOHcjLy8P9999fY8l/8+bN0aVLF5w8eVK3qk3rnXfeQefOnfHAAw+gd+/e8PPzq9Go8t8+//xzBAcH495778Vjjz2GyZMnw8HBQfe8g4MDdu7ciZCQEDzyyCNo3bo1xowZg5KSEo4sEZkJQfz3pDoRERERcSSJiIiISB8mSURERER6MEkiIiIi0oNJEhEREZEeTJKIiIiI9GCSRERERKQHkyQiIiIiPZgkEREREenBJImIiIhIDyZJRERERHowSSIiIiLSg0kSERERkR7/D2CmW4P1b2O3AAAAAElFTkSuQmCC",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Display the Accuracy Score Curve graph.\n",
        "\n",
        "plt.plot(range(1, 11), accuracy_scores)\n",
        "plt.xlabel('K-value')\n",
        "plt.ylabel('Accuracy Score')\n",
        "plt.title('Accuracy Score Curve')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vPk88dYXYFTA",
        "outputId": "643ab2c5-916b-49d8-e2c7-fecd5de32a21"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>k</th>\n",
              "      <th>accuracy_score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>0.562042</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0.740319</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>0.718634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>0.672117</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>0.562042</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>0.664479</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>7</td>\n",
              "      <td>0.663942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>8</td>\n",
              "      <td>0.663692</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>9</td>\n",
              "      <td>0.623028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>10</td>\n",
              "      <td>0.657379</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    k  accuracy_score\n",
              "0   1        0.562042\n",
              "1   2        0.740319\n",
              "2   3        0.718634\n",
              "3   4        0.672117\n",
              "4   5        0.562042\n",
              "5   6        0.664479\n",
              "6   7        0.663942\n",
              "7   8        0.663692\n",
              "8   9        0.623028\n",
              "9  10        0.657379"
            ]
          },
          "execution_count": 81,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Construct a data frame with the K-value and the corresponding accuracy score.\n",
        "\n",
        "kmeans_accuracy_df = pd.DataFrame({'k': range(1, 11), 'accuracy_score': accuracy_scores})\n",
        "kmeans_accuracy_df"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "78mAWlYwYFTA"
      },
      "source": [
        "The best cluster size is **n_clusters=2** with an accuracy of 74.03%."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "GWqzujPO0iT3"
      },
      "source": [
        "# [ RESULTS ]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "MDzEJzcj0iT3"
      },
      "source": [
        "# 6 | Results and Discussions"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "XUqwSRHa0iT3"
      },
      "source": [
        "## 6.1 | Existing Solutions for the 'Toxic Speech Classifier' problem."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "SGHtx4ko_iGK"
      },
      "source": [
        "The [**Perspective API**](https://perspectiveapi.com/) exposed by [Firebase Extensions](https://extensions.dev/extensions/jigsaw/firestore-perspective-toxicity) (BaaS - Backend as a Service) is a commercialize toxic speech classifier developed by the [Jigsaw](https://jigsaw.google.com/) company.\n",
        "\n",
        "![Perspective API in the Firebase Extensions Hub](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/sample1_perspective_api.png?raw=true)\n",
        "\n",
        "The downside is that this toxic classifier only supports **few** languages (e.g., it does not support the Filipino language). That is why, we decided to make our own classifier (this project) that supports Filipino and Taglish (aside from English) so that we can use it locally here in our country - Philippines.\n",
        "\n",
        "Many other toxic classfiers, hate speech classifiers, profanity classifiers, and the like exist in the ever booming market of SaaS and AIaaS digital products. But why they do all have in common is that they don't support the Filipino language (and its dialects - Ilokano, Bicolano, 'TagLish', etc.)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "OthLO-3F0iT3"
      },
      "source": [
        "## 6.2 | Monomial Naive Bayes Classifier"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "LnraOFDb0iT3"
      },
      "source": [
        "MNB accuracy rate *before* hypertuning the parameters: **~86.379%**\n",
        "\n",
        "![MNB without optimized parameters - Confusion Matrix](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/MNB_without_optimized_params.png?raw=true)\n",
        "![MNB without optimized parameters - Classification Report](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/MNB_without_optimized_params_classification_report.png?raw=true)\n",
        "\n",
        "\n",
        "MNB accuracy rate *after* hypertuning the parameters: **~87.093%**\n",
        "\n",
        "![MNB with optimized parameters - Confusion Matrix](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/MNB_with_optimized_params.png?raw=true)\n",
        "![MNB without optimized parameters - Classification Report](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/MNB_with_optimized_params_classification_report.png?raw=true)\n",
        "\n",
        "There is an improvement of **0.714%**.\n",
        "\n",
        "Looking at the confusion matrices of both models, we see that both the True Negatives and True Positives have a higher value while the False Negatives and False Positives have a lower value in the confusion matrix of the MNB with optimized parameters. This is the cause of the increase in accuracy rate in the optimized MNB model. Looking at the classification report, we also see that consequentially, all the values regarding precision, recall, and f1-score have a higher value in the optimized MNB model. This means that hypertuning the model was able to greatly aid in accuracy.\n",
        "\n",
        "For the remaining parts of the discussion, the MNB model will refer to the **optimized MNB model** since it has a higher accuracy score.\n",
        "\n",
        "An interesting aspect to look at for the increase in accuracy score after tuning are the parameters of the model, namely the customized list of stop words. The 'hypertuned' stop words include common tokens, both in English and Tagalog, as well as some conjunctions and filler words.\n",
        "\n",
        "A reason for this would be that these words are unnecessarily used by many Twitter users and do not add to the context of many sentences. The model was tuned to look at the words that contribute context to the sentence, thereby filtering out numbers and filler words. Focusing on the non-connecting words of a tweet instead of filler words to make the sentence gramatically correct lets the model obtain the main idea of the tweet much more accurately."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "FlyEqcRG0iT3"
      },
      "source": [
        "## 6.3 | KMeans Clustering"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xPsboVLm0xny"
      },
      "source": [
        "KMeans accuracy rate *before* using tSNE (n_clusters=2): **~67.519%**\n",
        "\n",
        "![MNB without tSNE](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/KMeans_without_tSNE.png?raw=true)\n",
        "\n",
        "\n",
        "KMeans accuracy rate *after* use tSNE (n_clusters=2): **~74.032%**\n",
        "\n",
        "![MNB with tSNE](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/KMeans_with_tSNE.png?raw=true)\n",
        "\n",
        "There is an improvement of **6.513%**.\n",
        "\n",
        "From the confusion matrices of both K-means models, we see that the model without using tSNE had a higher value for True Negatives, classifying non-toxic tweets as non-toxic, and a lower value for False Positives, classifying non-toxic tweets as toxic, when compare to the K-means model that uses tSNE. This shows that the model without using tSNE is more efficient in classifying tweets that are non-toxic rather than identifying tweets that are toxic.\n",
        "\n",
        "This model is a highly conservative model that has strict implementations on classifying a tweet as toxic. This is proven by the high number of False Negatives as seen in the confusion matrix. Though the high number of True Negatives is appealing, the aim of the project is to identify toxic tweets rather than non-toxic tweets, therefore the model is not ideal for the objectives of the project.\n",
        "\n",
        "Looking at the confusion matrix of the K-means model where tSNE was applied, we see that the model has a lower rate of classifying non-toxic tweets correctly, in exchange for correctly classifying toxic tweets. This model is more ideal towards the project's objectives and has a higher accuracy rate. Therefore, the tSNE K-means model is better, with regards to the scope of the project.\n",
        "\n",
        "Moving forward, the K-means model will refer to the K-means model where tSNE was applied."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Tgx7SEqf-AEi"
      },
      "source": [
        "We used a k-mean value of **2** since it gives the best accuracy score (as can be seen in the **Elbow Curve** and **Accuracy Score Curve**). This makes sense since the dataset is primarily separated by **two distinct** classes afterall (i.e., non-toxic and toxic class).\n",
        "\n",
        "![Elbow Curve](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/KMeans_elbow_curve.png?raw=true)\n",
        "![Accuraccy Score Curve](https://github.com/Qubits-01/toxic-speech-classifier/blob/main/assets/KMeans_accurarcy_score_curve.png?raw=true)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "PTItbBLK0iT3"
      },
      "source": [
        "## 6.4 | Multinomial Naive Bayes vs K-means Model"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Cp14cd53_dBO"
      },
      "source": [
        "MNB accuracy rate *after* hypertuning the parameters: **~87.093%**\n",
        "\n",
        "KMeans accuracy rate *after* use tSNE (n_clusters=2): **~74.032%**\n",
        "\n",
        "Since the size of the data used for evaluating the model is different between the two, it is best to use the accuracy rate as a basis for comparison. The MNB model shows an accuracy rate of roughly 87% while the Kmeans model shows an accuracy rate of almost 74%. The difference between the two is largely in part of the nature of the models itself.\n",
        "\n",
        "The MNB classification algorithm is a classification algorithm that is highly effective for text classification, which includes but is not limited to: spam filtering, sentiment analysis, and topic modeling. This makes it easy and effective to use on text classification similar to the objectives of the project.\n",
        "\n",
        "The Kmeans algorithm is more efficient for dimensionality reduction and visualization, which is somewhat related to the project's objectives, if the tweets were looked at as 2 clusters which divides the toxic and non-toxic tweets. We also hypothesized that the kMeans model had a hard time 'classifying' the large amount of tokens in the data because there are a lot of noise and clutter (for instance there is no easy way to remove some stopwords  in the training data for the kMeans model). As a result, this reduces the accuracy rate of the kMeans model greatly.\n",
        "\n",
        "Thus, the wide gap between accuracy scores are valid and it makes sense that MNB would be more suited to the project.\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xC1EP4vsEcvJ"
      },
      "source": [
        "## 6.5 | Conclusion"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ZW0rk-fvEcvJ"
      },
      "source": [
        "With an accuracy of 87%, the hypertuned Multinomial Naive Bayes model is able to achieve the goals of the project, to classify tweets into the toxic and non-toxic categories (and to have an accuracy rate greater than 75%). Among the tested models, we conclude that it is the most efficient one that is able to produce satisfying results.\n",
        "\n",
        "For future studies, coming up with a more efficient or effective model may include using a different classification algorithm such as the Support Vector Machine algorithm, or to create a better set of optimized parameters by including a different set of stop words. \n",
        "\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "KH-vSLua0iT4"
      },
      "source": [
        "# 7 | Some Results Showcase"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "metadata": {
        "id": "_2CHUR3G0iT4",
        "outputId": "b7064e0b-626b-4998-c094-166f0b4a934a"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Di ako makapaniwala na pangalawa si Mar Roxas ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>7.552559e-03</td>\n",
              "      <td>9.924474e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>A lot of you niggas turned these bitches into ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.154667e-06</td>\n",
              "      <td>9.999988e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>RT @ZitlalyL: @Vic_Manzan0 just like you&amp;#1280...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4.166335e-10</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>our playground for the morning.  big bear tri ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.396200e-01</td>\n",
              "      <td>6.038000e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>#nazi #swastika #ebay! #trump #donaldtrump #re...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5.883882e-02</td>\n",
              "      <td>9.411612e-01</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17364</th>\n",
              "      <td>RT @ImNeverChillin: Niggas drop outta high sch...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3.443211e-11</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17365</th>\n",
              "      <td>Mad respect to Affleck for refusing to wear a ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.530769e-01</td>\n",
              "      <td>4.692311e-02</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17366</th>\n",
              "      <td>RT @TAXSTONE: If YA bitch ex is a nigga u thin...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>5.077755e-13</td>\n",
              "      <td>1.000000e+00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17368</th>\n",
              "      <td>fathersdaymessage  #fathers #day #message buy...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.000000e+00</td>\n",
              "      <td>2.774210e-11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17369</th>\n",
              "      <td>@user my woodpecker shot in this months issue...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9.933148e-01</td>\n",
              "      <td>6.685236e-03</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>15128 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "0      Di ako makapaniwala na pangalawa si Mar Roxas ...       1          1   \n",
              "1      A lot of you niggas turned these bitches into ...       1          1   \n",
              "2      RT @ZitlalyL: @Vic_Manzan0 just like you&#1280...       1          1   \n",
              "3      our playground for the morning.  big bear tri ...       0          0   \n",
              "5      #nazi #swastika #ebay! #trump #donaldtrump #re...       1          1   \n",
              "...                                                  ...     ...        ...   \n",
              "17364  RT @ImNeverChillin: Niggas drop outta high sch...       1          1   \n",
              "17365  Mad respect to Affleck for refusing to wear a ...       0          0   \n",
              "17366  RT @TAXSTONE: If YA bitch ex is a nigga u thin...       1          1   \n",
              "17368   fathersdaymessage  #fathers #day #message buy...       0          0   \n",
              "17369   @user my woodpecker shot in this months issue...       0          0   \n",
              "\n",
              "       prob_non_toxic    prob_toxic  \n",
              "0        7.552559e-03  9.924474e-01  \n",
              "1        1.154667e-06  9.999988e-01  \n",
              "2        4.166335e-10  1.000000e+00  \n",
              "3        9.396200e-01  6.038000e-02  \n",
              "5        5.883882e-02  9.411612e-01  \n",
              "...               ...           ...  \n",
              "17364    3.443211e-11  1.000000e+00  \n",
              "17365    9.530769e-01  4.692311e-02  \n",
              "17366    5.077755e-13  1.000000e+00  \n",
              "17368    1.000000e+00  2.774210e-11  \n",
              "17369    9.933148e-01  6.685236e-03  \n",
              "\n",
              "[15128 rows x 5 columns]"
            ]
          },
          "execution_count": 72,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the correctly classified text.\n",
        "\n",
        "test_results_df[(test_results_df['actual'] == test_results_df['predicted']) & ((test_results_df['prob_toxic'] > 0.5) | (test_results_df['prob_non_toxic'] > 0.5))]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {
        "id": "Pg00qn2Y0iT4",
        "outputId": "7e9857bb-4dd9-4b3b-89a8-aa93eec2ee46"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"@ellisluciano: Binay as President #ShortHorro...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.709684</td>\n",
              "      <td>0.290316</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>Anyone want to share their thoughts on their m...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.672222</td>\n",
              "      <td>0.327778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>\"..a broader phenomenon: the reemergence of #r...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.975314</td>\n",
              "      <td>0.024686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>Nakakabeast mode naman. Di ka pa nga botante, ...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.639222</td>\n",
              "      <td>0.360778</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>joshwin is always like this! ðð what's w...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.283788</td>\n",
              "      <td>0.716212</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17337</th>\n",
              "      <td>expes explain the best way to deal with  frien...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.999352</td>\n",
              "      <td>0.000648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17338</th>\n",
              "      <td>this! #cambersands deaths blamed on victims' \"...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.957978</td>\n",
              "      <td>0.042022</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17339</th>\n",
              "      <td>RT @Elisaa_Martinez: Happy birthday cotton pic...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.001537</td>\n",
              "      <td>0.998463</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17343</th>\n",
              "      <td>Hindi ko maintindihan bat nasa fifth spot si S...</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.906055</td>\n",
              "      <td>0.093945</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17367</th>\n",
              "      <td>@imcessfederico trillion yan ! Wala eh... Only...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0.087168</td>\n",
              "      <td>0.912832</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>2242 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "4      \"@ellisluciano: Binay as President #ShortHorro...       1          0   \n",
              "31     Anyone want to share their thoughts on their m...       1          0   \n",
              "32     \"..a broader phenomenon: the reemergence of #r...       1          0   \n",
              "35     Nakakabeast mode naman. Di ka pa nga botante, ...       1          0   \n",
              "45     joshwin is always like this! ðð what's w...       0          1   \n",
              "...                                                  ...     ...        ...   \n",
              "17337  expes explain the best way to deal with  frien...       1          0   \n",
              "17338  this! #cambersands deaths blamed on victims' \"...       1          0   \n",
              "17339  RT @Elisaa_Martinez: Happy birthday cotton pic...       0          1   \n",
              "17343  Hindi ko maintindihan bat nasa fifth spot si S...       1          0   \n",
              "17367  @imcessfederico trillion yan ! Wala eh... Only...       0          1   \n",
              "\n",
              "       prob_non_toxic  prob_toxic  \n",
              "4            0.709684    0.290316  \n",
              "31           0.672222    0.327778  \n",
              "32           0.975314    0.024686  \n",
              "35           0.639222    0.360778  \n",
              "45           0.283788    0.716212  \n",
              "...               ...         ...  \n",
              "17337        0.999352    0.000648  \n",
              "17338        0.957978    0.042022  \n",
              "17339        0.001537    0.998463  \n",
              "17343        0.906055    0.093945  \n",
              "17367        0.087168    0.912832  \n",
              "\n",
              "[2242 rows x 5 columns]"
            ]
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show the misclassified text.\n",
        "\n",
        "test_results_df[(test_results_df['actual'] != test_results_df['predicted'])]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "id": "GitH9-g70iT4",
        "outputId": "aa8c889c-77e6-4c3d-af20-5c8c026e87d8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>13532</th>\n",
              "      <td>Stern fixed the image of the league.. NBA play...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.049795</td>\n",
              "      <td>0.950205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9314</th>\n",
              "      <td>@nCrusade @GodManiac_ come wage with us fag</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.049670</td>\n",
              "      <td>0.950330</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9106</th>\n",
              "      <td>shet ibabalik yung martial law at take note ma...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.049488</td>\n",
              "      <td>0.950512</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12506</th>\n",
              "      <td>@cnnphilippines Good job, Mrs. Poe! But Dutert...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.049032</td>\n",
              "      <td>0.950968</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8479</th>\n",
              "      <td>@MoenKiana I am working fag</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.049030</td>\n",
              "      <td>0.950970</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15432</th>\n",
              "      <td>Ugh! Another annoying from VP Binay. ??</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048755</td>\n",
              "      <td>0.951245</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2352</th>\n",
              "      <td>magsama sila ni binay. Mukhang pera.</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048528</td>\n",
              "      <td>0.951472</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6999</th>\n",
              "      <td>Ate M: Pinatunayan ni nognog na may mas maitim...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048387</td>\n",
              "      <td>0.951613</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10094</th>\n",
              "      <td>Wow my dad just racked me what a fag</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048249</td>\n",
              "      <td>0.951751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9587</th>\n",
              "      <td>RT @BeYouTiful_Pris: Recognize that I'm a star...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048197</td>\n",
              "      <td>0.951803</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17149</th>\n",
              "      <td>Bakit top 2 si Mar Roxas? Pakiexplain!!! #Daya...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.048146</td>\n",
              "      <td>0.951854</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16737</th>\n",
              "      <td>Wow naman Mar Roxas, napaka hardworking mo nam...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047980</td>\n",
              "      <td>0.952020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4701</th>\n",
              "      <td>Binay, you are so fucking desperate.</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047815</td>\n",
              "      <td>0.952185</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11824</th>\n",
              "      <td>Yan na naman ang walang kwentang commercial ni...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047572</td>\n",
              "      <td>0.952428</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12460</th>\n",
              "      <td>RT @Comp10Tx: Rihanna really is trash</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047457</td>\n",
              "      <td>0.952543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2507</th>\n",
              "      <td>eto na naman si Bakit kay Binay? bwiset @abcde...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047443</td>\n",
              "      <td>0.952557</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2277</th>\n",
              "      <td>@user fuck you you #homophobic  #misogynistic ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047272</td>\n",
              "      <td>0.952728</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6928</th>\n",
              "      <td>@carlomer ahahahaha. Di kasi makikita si binay...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047249</td>\n",
              "      <td>0.952751</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10955</th>\n",
              "      <td>Pinaka korning commercial yung kay Mar Roxas.??</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047133</td>\n",
              "      <td>0.952867</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1964</th>\n",
              "      <td>@RespectBox @lefthooklounge @Rodriguez5_ he's ...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.047044</td>\n",
              "      <td>0.952956</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "13532  Stern fixed the image of the league.. NBA play...       1          1   \n",
              "9314         @nCrusade @GodManiac_ come wage with us fag       1          1   \n",
              "9106   shet ibabalik yung martial law at take note ma...       1          1   \n",
              "12506  @cnnphilippines Good job, Mrs. Poe! But Dutert...       1          1   \n",
              "8479                         @MoenKiana I am working fag       1          1   \n",
              "15432            Ugh! Another annoying from VP Binay. ??       1          1   \n",
              "2352                magsama sila ni binay. Mukhang pera.       1          1   \n",
              "6999   Ate M: Pinatunayan ni nognog na may mas maitim...       1          1   \n",
              "10094               Wow my dad just racked me what a fag       1          1   \n",
              "9587   RT @BeYouTiful_Pris: Recognize that I'm a star...       1          1   \n",
              "17149  Bakit top 2 si Mar Roxas? Pakiexplain!!! #Daya...       1          1   \n",
              "16737  Wow naman Mar Roxas, napaka hardworking mo nam...       1          1   \n",
              "4701                Binay, you are so fucking desperate.       1          1   \n",
              "11824  Yan na naman ang walang kwentang commercial ni...       1          1   \n",
              "12460              RT @Comp10Tx: Rihanna really is trash       1          1   \n",
              "2507   eto na naman si Bakit kay Binay? bwiset @abcde...       1          1   \n",
              "2277   @user fuck you you #homophobic  #misogynistic ...       1          1   \n",
              "6928   @carlomer ahahahaha. Di kasi makikita si binay...       1          1   \n",
              "10955    Pinaka korning commercial yung kay Mar Roxas.??       1          1   \n",
              "1964   @RespectBox @lefthooklounge @Rodriguez5_ he's ...       1          1   \n",
              "\n",
              "       prob_non_toxic  prob_toxic  \n",
              "13532        0.049795    0.950205  \n",
              "9314         0.049670    0.950330  \n",
              "9106         0.049488    0.950512  \n",
              "12506        0.049032    0.950968  \n",
              "8479         0.049030    0.950970  \n",
              "15432        0.048755    0.951245  \n",
              "2352         0.048528    0.951472  \n",
              "6999         0.048387    0.951613  \n",
              "10094        0.048249    0.951751  \n",
              "9587         0.048197    0.951803  \n",
              "17149        0.048146    0.951854  \n",
              "16737        0.047980    0.952020  \n",
              "4701         0.047815    0.952185  \n",
              "11824        0.047572    0.952428  \n",
              "12460        0.047457    0.952543  \n",
              "2507         0.047443    0.952557  \n",
              "2277         0.047272    0.952728  \n",
              "6928         0.047249    0.952751  \n",
              "10955        0.047133    0.952867  \n",
              "1964         0.047044    0.952956  "
            ]
          },
          "execution_count": 74,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Show some toxic text that were correctly classified.\n",
        "\n",
        "test_results_df[(test_results_df['actual'] == test_results_df['predicted']) & (test_results_df['predicted'] == 1) &(test_results_df['prob_toxic'] > 0.95)].sort_values(by='prob_toxic').head(20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {
        "id": "bk9z-2UIYFS-",
        "outputId": "b7b96d98-7877-4715-bf3d-d480261c9fa8"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>actual</th>\n",
              "      <th>predicted</th>\n",
              "      <th>prob_non_toxic</th>\n",
              "      <th>prob_toxic</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3870</th>\n",
              "      <td>this time they did it #belgium  ððð</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950027</td>\n",
              "      <td>0.049973</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6678</th>\n",
              "      <td>@user the only person behaving as an idiot wo...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950081</td>\n",
              "      <td>0.049919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11938</th>\n",
              "      <td>#RoxasRobredo</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950092</td>\n",
              "      <td>0.049908</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8827</th>\n",
              "      <td>@GabeJGonzalez @RPopBox and a prime Leija woul...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950168</td>\n",
              "      <td>0.049832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9335</th>\n",
              "      <td>5 proven #habits that   people have</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950203</td>\n",
              "      <td>0.049797</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6906</th>\n",
              "      <td>Great picture, could stand for Binay, Marcos, ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950307</td>\n",
              "      <td>0.049693</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3763</th>\n",
              "      <td>i was so pumped cause bones will be having fou...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950412</td>\n",
              "      <td>0.049588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13686</th>\n",
              "      <td>can you guess what i did friday?ðð½ #barn...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950553</td>\n",
              "      <td>0.049447</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14210</th>\n",
              "      <td>smiling, cheering, crying ... #americanninjawa...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950559</td>\n",
              "      <td>0.049441</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13800</th>\n",
              "      <td>For Mar Roxas ? think 100x Duterte Solid to! h...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950687</td>\n",
              "      <td>0.049313</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8074</th>\n",
              "      <td>some   #customers at the #brewery. :)</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950693</td>\n",
              "      <td>0.049307</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6132</th>\n",
              "      <td>#newcomeback #stellar #newconcept i'm excited</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.950737</td>\n",
              "      <td>0.049263</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15873</th>\n",
              "      <td>@user it's been #bad for a while now</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951049</td>\n",
              "      <td>0.048951</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14042</th>\n",
              "      <td>wawo nic gud  bless  you</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951087</td>\n",
              "      <td>0.048913</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1937</th>\n",
              "      <td>Here's what Mar Roxas has to say about  https:...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951106</td>\n",
              "      <td>0.048894</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9492</th>\n",
              "      <td>RT @profmssyj: did u seek audit of Binay befor...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951152</td>\n",
              "      <td>0.048848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14809</th>\n",
              "      <td>Binay’s birthday wish: Poverty alleviation for...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951275</td>\n",
              "      <td>0.048725</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11015</th>\n",
              "      <td>@user wtf this is gross</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951385</td>\n",
              "      <td>0.048615</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9567</th>\n",
              "      <td>Rose colored glasses only, you guys. #wvnextin6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951540</td>\n",
              "      <td>0.048460</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3871</th>\n",
              "      <td>#RoxasRising https://t.co/XHp3YNpv61</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.951561</td>\n",
              "      <td>0.048439</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                    text  actual  predicted  \\\n",
              "3870        this time they did it #belgium  ððð       0          0   \n",
              "6678    @user the only person behaving as an idiot wo...       0          0   \n",
              "11938                                      #RoxasRobredo       0          0   \n",
              "8827   @GabeJGonzalez @RPopBox and a prime Leija woul...       0          0   \n",
              "9335                5 proven #habits that   people have        0          0   \n",
              "6906   Great picture, could stand for Binay, Marcos, ...       0          0   \n",
              "3763   i was so pumped cause bones will be having fou...       0          0   \n",
              "13686  can you guess what i did friday?ðð½ #barn...       0          0   \n",
              "14210  smiling, cheering, crying ... #americanninjawa...       0          0   \n",
              "13800  For Mar Roxas ? think 100x Duterte Solid to! h...       0          0   \n",
              "8074              some   #customers at the #brewery. :)        0          0   \n",
              "6132    #newcomeback #stellar #newconcept i'm excited          0          0   \n",
              "15873             @user it's been #bad for a while now         0          0   \n",
              "14042                          wawo nic gud  bless  you        0          0   \n",
              "1937   Here's what Mar Roxas has to say about  https:...       0          0   \n",
              "9492   RT @profmssyj: did u seek audit of Binay befor...       0          0   \n",
              "14809  Binay’s birthday wish: Poverty alleviation for...       0          0   \n",
              "11015                           @user wtf this is gross        0          0   \n",
              "9567     Rose colored glasses only, you guys. #wvnextin6       0          0   \n",
              "3871                #RoxasRising https://t.co/XHp3YNpv61       0          0   \n",
              "\n",
              "       prob_non_toxic  prob_toxic  \n",
              "3870         0.950027    0.049973  \n",
              "6678         0.950081    0.049919  \n",
              "11938        0.950092    0.049908  \n",
              "8827         0.950168    0.049832  \n",
              "9335         0.950203    0.049797  \n",
              "6906         0.950307    0.049693  \n",
              "3763         0.950412    0.049588  \n",
              "13686        0.950553    0.049447  \n",
              "14210        0.950559    0.049441  \n",
              "13800        0.950687    0.049313  \n",
              "8074         0.950693    0.049307  \n",
              "6132         0.950737    0.049263  \n",
              "15873        0.951049    0.048951  \n",
              "14042        0.951087    0.048913  \n",
              "1937         0.951106    0.048894  \n",
              "9492         0.951152    0.048848  \n",
              "14809        0.951275    0.048725  \n",
              "11015        0.951385    0.048615  \n",
              "9567         0.951540    0.048460  \n",
              "3871         0.951561    0.048439  "
            ]
          },
          "execution_count": 75,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#  Show some non-toxic text that were correctly classified.\n",
        "\n",
        "test_results_df[(test_results_df['actual'] == test_results_df['predicted']) & (test_results_df['predicted'] == 0) &(test_results_df['prob_non_toxic'] > 0.95)].sort_values(by='prob_non_toxic').head(20)\n"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "ARpxplt6YFTA"
      },
      "source": [
        "# 8 | For Bonus Points (AIaaS - AI as a service)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "SW1v5U0-fkw_"
      },
      "source": [
        "The generated files by the code snippets below will be later used by the backend code of this project. This is useful in creating an API for the AI model that was generated/built in this project. This is inline with our goal in converting the results/outputs of this project into a working AIaaS digital  product (AI as a Service) that can be freely used by mobile apps, web apps, and the like through http/https requests."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 76,
      "metadata": {
        "id": "mr6RbPsnYFTA",
        "outputId": "0e970b62-3a6a-4588-8fcc-4e92df15ae33"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['./backend/mnb_model.joblib']"
            ]
          },
          "execution_count": 76,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Save the fitted Monomial Naive Bayes model locally.\n",
        "# This will be used for the FastAPI backend.\n",
        "\n",
        "import joblib\n",
        "\n",
        "joblib.dump(mnb, './backend/mnb_model.joblib')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 77,
      "metadata": {
        "id": "lrqnfuEBYFTB"
      },
      "outputs": [],
      "source": [
        "# Save the X_train data frame locally.\n",
        "# This will be used for the FastAPI backend.\n",
        "# (i.e., to fit the CountVectorizer instance there).\n",
        "\n",
        "X_train.to_csv('./backend/X_train.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 78,
      "metadata": {
        "id": "CwrCuymhCk_b"
      },
      "outputs": [],
      "source": [
        "# Save the custom stopwords list to a text file locally.\n",
        "# This will be used for the FastAPI backend.\n",
        "\n",
        "with open('./backend/custom_stopwords.txt', 'w') as file:\n",
        "    for word in custom_stop_words:\n",
        "        file.write(word + '\\n')\n",
        "\n",
        "# Close the file.\n",
        "file.close()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 0
}