05-Try-out-Spacy-Pretrain.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spacy Classification Model from Scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pd.set_option('max_colwidth',1000)\n",
    "pd.options.display.float_format = '{:.2f}'.format\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read in file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tomlin/miniconda3/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3020: DtypeWarning: Columns (5,11) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 66806 entries, 190126 to 553096\n",
      "Data columns (total 18 columns):\n",
      "date_received                   66806 non-null object\n",
      "product                         66806 non-null object\n",
      "sub_product                     46351 non-null object\n",
      "issue                           66806 non-null object\n",
      "sub_issue                       32932 non-null object\n",
      "consumer_complaint_narrative    66806 non-null object\n",
      "company_public_response         32776 non-null object\n",
      "company                         66806 non-null object\n",
      "state                           66620 non-null object\n",
      "zipcode                         66617 non-null object\n",
      "tags                            11417 non-null object\n",
      "consumer_consent_provided       66806 non-null object\n",
      "submitted_via                   66806 non-null object\n",
      "date_sent_to_company            66806 non-null object\n",
      "company_response_to_consumer    66806 non-null object\n",
      "timely_response                 66806 non-null object\n",
      "consumer_disputed?              66806 non-null object\n",
      "complaint_id                    66806 non-null int64\n",
      "dtypes: int64(1), object(17)\n",
      "memory usage: 9.7+ MB\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "None"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dir_path = '../Playground-dataset/05-Try-out-Spacy-Pretrain-dataset'\n",
    "\n",
    "df_raw = pd.read_csv(os.path.join(dir_path, 'consumer_complaints.csv'))\n",
    "df_raw = df_raw[pd.notnull(df_raw['consumer_complaint_narrative'])]\n",
    "\n",
    "# How's the dataset like?\n",
    "display(df_raw.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter on Text and Label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter on text and label.\n",
    "df = df_raw[['product','consumer_complaint_narrative']]\n",
    "\n",
    "\n",
    "# Experiment on smaller subset.\n",
    "df = df[:5000]\n",
    "df = df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frequency(Top 30)...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>freq</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Debt collection</th>\n",
       "      <td>0.28</td>\n",
       "      <td>1423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mortgage</th>\n",
       "      <td>0.23</td>\n",
       "      <td>1171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Credit reporting</th>\n",
       "      <td>0.16</td>\n",
       "      <td>811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Credit card</th>\n",
       "      <td>0.11</td>\n",
       "      <td>547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bank account or service</th>\n",
       "      <td>0.08</td>\n",
       "      <td>403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consumer Loan</th>\n",
       "      <td>0.05</td>\n",
       "      <td>268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Student loan</th>\n",
       "      <td>0.04</td>\n",
       "      <td>219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Payday loan</th>\n",
       "      <td>0.01</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Money transfers</th>\n",
       "      <td>0.01</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Prepaid card</th>\n",
       "      <td>0.01</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other financial service</th>\n",
       "      <td>0.00</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         freq  count\n",
       "Debt collection          0.28   1423\n",
       "Mortgage                 0.23   1171\n",
       "Credit reporting         0.16    811\n",
       "Credit card              0.11    547\n",
       "Bank account or service  0.08    403\n",
       "Consumer Loan            0.05    268\n",
       "Student loan             0.04    219\n",
       "Payday loan              0.01     65\n",
       "Money transfers          0.01     54\n",
       "Prepaid card             0.01     31\n",
       "Other financial service  0.00      8"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4MAAAIlCAYAAABvpS29AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XtcVHX+x/E3DBcVKC+rloZS2LCokXfdDO+6W6FZ+St1M9u8VpqZGWWarLqImWXWYqmFYZlo1lp52Z+3FSKzIkkltGK9ZbteQncB4zrn90c/Z2XRnNkdZpDv6/l49Hgw58yc8z6f0Jm358yMn2VZlgAAAAAARvH3dQAAAAAAgPdRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwEABvg4AAPC+Pn366NixYxdcl5OTo4AAnh585bvvvlPfvn0vuj4lJUU33XTTf7WPJ598UmfPntWiRYvcetyuXbu0bNky7dmzRxUVFYqKitL48eMVGxvrXH/ffffpiy++UEhIyH+VEQBQ/Xi2BwBDPfbYY7rzzjurLKcI1gwrVqzQtddeW2X5lVde6YM00urVq/WHP/xB48eP19SpU2Wz2fTBBx9o7NixWrBggW699Vaf5AIA/Od4xgcAQ4WEhKhx48a+joGLqF+/fo35/3Ps2DHNmTNHv//973XHHXc4lz/66KM6ffq0kpKS1L9/fx8mBAD8J3jPIACgipdeekmjR4/WqFGj1LFjR7333nuSpNdee029e/dW+/btNWzYMGVnZzsfU15ernnz5qlbt27q2rWrlixZojvvvFPvvvuuJGnEiBGaN29epf1ERUVp+/btzts/t/0nn3xSCQkJeuqpp9S+fXv16dNHycnJzvUVFRX64x//qF69eql9+/YaMWKEvvnmG504cUKtW7fWzp07K+27f//+Wrt27QWPf+/evbrvvvvUvn173XzzzZo/f77KyspkWZZ69+6t1NTUSvd/6KGHNGPGDEnSyZMnNWnSJOdjn376aRUUFEj66RLQqKgoJScnq2vXrhozZoxr/0Mu4NSpU5oyZYq6deumtm3bql+/flqzZo1zfXFxsRITE9W9e3d17NhRDz74oI4fP+5c/+OPP2ratGnq0KGDunXrphdeeOGi+/rwww915ZVX6vbbb6+ybsKECUpOTpbNZquy7tChQxo/frw6deqktm3bKi4uTlu3bnWu37x5s+Li4nTDDTeoT58+WrZsmXPd559/riFDhigmJkY333yznn32WVVUVLg9JwDAxVEGAQAXlJGRoU6dOmn16tXq2bOnVq1apdTUVM2cOVPvvfeeevbsqZEjR+ro0aOSfiqQ69at07x587R8+XJt27ZN33zzjcv7u9T2Jemdd95R06ZNtXbtWg0ZMkQvvvii9u3bJ0n64x//qBUrVmjatGl67733dPXVV2vcuHFq1KiRunXrpg8//NC5nezsbB0/flwDBgyokuPgwYMaMWKEWrVqpXfeeUezZ8/WunXr9Pzzz8vPz0+33nqrNm7c6Lx/QUGBMjIyNHDgQEnSxIkTZVmW0tLStHjxYh05ckSTJ0+utI+//OUvWrVqlZ544gmX5/Pv4uPjlZ+fr+XLl2v9+vXq06ePEhISdPLkSUnSzJkztXXrVs2fP1+rV69WaWmpHn30Uefj09PT1bhxY61bt05TpkzRK6+8ooyMjAvuKzc3V23atJG/f9WXDY0bN1bbtm2rrLMsS+PHj1dISIjS0tK0bt062e12TZs2TaWlpTp16pQmT56skSNHatOmTXriiSe0cOFC7dy5UxUVFXr44YfVq1cvbdiwQc8++6zWrFnj/IcFAICHWAAA4/Tu3dtq06aN1a5du0r/ffLJJ5ZlWdaiRYusmJgYq6KiwvmYXr16WevWrau0nd/97ndWUlKS5XA4rG7dulmpqanOdUePHrWio6OttWvXWpZlWffee6+VlJRU6fF2u93atm3bJbdvWZYVHx9v/eY3v6m0vkuXLtaqVauc+1++fLlzXUFBgTV37lzrhx9+sN577z2rc+fOVmlpqWVZljV79mxr4sSJF5xNUlKSFRcXZzkcDuey9evXW23atLGKioqs3NxcKyoqyvr73/9uWZZlvfvuu1aPHj0sh8Nh7dy502rXrp1VUlLifOzf//53y263WwcOHLCOHj1q2e1268MPP7zgvs/NzW63WzExMVX+/9x6663O+73xxhvWkSNHnLfz8/Mtu91uffbZZ9Y///lPq3Xr1tbmzZud648cOWLNnz/fKikpseLj4624uLhK++3Xr5+1ZMmSC2b63e9+Z02ZMuWimc/55JNPLLvdbhUWFlpnz561li1bZp0+fdq5fu/evZbdbre+//57Kycnx7Lb7VZGRoZz/WeffWadPHnSOn36tGW32620tDTnui+//NL67rvvLpkBAOA63jMIAIYaN26cBg0aVGlZ06ZNnT9fc801zrM9RUVF+v777zVjxgzNnDnTeZ/S0lIFBQUpPz9f+fn5atOmTaXHX3XVVS5ludT2z2nZsmWlx4WEhKi8vFynT59Wfn6+brjhBue60NBQPfnkk5J+uiQ0ISFBH330kXr06KGNGzfq97///QWzfPPNN7rxxhvl5+fnXNaxY0eVlZXp8OHDio6O1vXXX6+NGzfq/vvv1/r163XbbbfJz89P3377rX788Ud17dq1ynYPHjzonE94ePglZ7Jo0SJFRERUWnb+h/sMGzZMmzZtUkpKig4dOqSvvvpK0k+Xyx48eFDl5eWV5hEeHq7HH3/cebtFixaVth0WFqaSkpILZmnQoIH++c9/XjLz+erWrathw4bpww8/1L59+3Tw4MFKGaOjozVw4ECNGjVK4eHh6tWrl26//Xb94he/kPTT7+eMGTP00ksvqWfPnrrtttsUExPjVgYAwM+jDAKAoRo0aFClXJ0vODjY+bPD4ZAkJSUlqXXr1pXuV6dOHdWpU+eC2zi/yP278vJyl7f/c9uzLEuBgYGSVKnAnS8kJER9+/bVhg0bFBQUpNLSUvXo0eOC973QsViWVSlnXFycNm3apMGDB2vnzp2aMmWK85iaNWumlJSUKtto1KiRzpw5c9F9/Lurr776ov9/LMvSqFGjdOLECd1222361a9+pVatWuk3v/mNpH/N6WLzkHTB9/idO85/FxMToyVLlsiyrCrbzMvL0x/+8Icq5frs2bO6++67FRwcrP79+6tPnz6qV6+eRowY4cz23HPPadSoUdq6dat27NihlStXKjExUYMHD9Zjjz2mO+64w7nugQce0KRJkzR+/PiLHhMAwD28ZxAAcElhYWFq3Lixjh8/rpYtWzr/e/PNN5WRkaGQkBA1b9680ge+nD59utJ3GQYFBTk/SEVSpfcCXmr7ruRr1KiR88yT9NMHqNx000368ssvJUmDBg1Senq6tmzZot/85jcXLaqRkZH68ssvKxWjL774QoGBgc6zaXFxcdqzZ4/WrFmjiIgIRUdHOx974sQJhYSEOI8hMDBQSUlJys/Pv+RxuOrbb7/Vrl27tHTpUk2cOFH9+/d3ztayLF1zzTWy2WyV5nHs2DF16dKl0ofIuOqWW25RUVGR84OEzrdixQrt37+/ylngTz/9VIcOHdLKlSs1fvx49erVS6dOnXJm3L9/v+bOnavo6GhNmDBBa9as0S233KL169fr+++/18yZM9W8eXONHj1aK1as0JgxY/T++++7nR0AcHGcGQQAuGT06NFKTk5WkyZNdMMNN+iDDz7QW2+9pRUrVkiSxo4dq+eff17h4eG69tpr9dxzz6m0tNT5+LZt2yo1NVWDBw9WaGiokpKSKhWyS23/Uu6//34lJyerefPmioiI0CuvvKKwsDBnUbv55psVEBCgtWvXVvrUyn83fPhwpaamavbs2frtb3+r7777TvPmzdPgwYMVFhYmSWrevLluvPFGJScna+zYsc7Hdu/eXddff70mT56s+Ph4BQQEaPbs2frHP/6h5s2b629/+5vL8z5z5ozzw2DOV69ePV1xxRWy2Wxav369Bg4cqEOHDmnOnDmSfrq0NjQ0VEOGDNHcuXMVEhKiBg0aKCkpSVFRUZUuBXZVkyZN9Pjjj+uZZ55xfvBOWVmZ3nvvPaWlpenFF190np09p379+iorK9OGDRvUpUsXffXVV0pMTHRmvPLKK/X222/riiuu0KBBg3TixAllZ2fr9ttvV8OGDfXnP/9ZkvTAAw+osLBQmZmZlS57BQD89yiDAACX3HfffSouLtb8+fN16tQpRUREaNGiRerYsaMkaejQoTpz5owSEhJUUlKiUaNGaffu3c7HP/DAA/rrX/+qUaNGqUGDBnrkkUcqnTm81PYvZdSoUSoqKtL06dNVVFSkDh066NVXX3UWTpvNpltuuUWbN29W586dL7qdpk2batmyZZo/f75uv/12NWjQQHfeeacefvjhSvcbOHCgvvjiC8XFxTmX+fv7a/HixfrDH/6g++67T/7+/uratavmz59/wcsyf865yyn/3YQJEzRx4kTNmjVLycnJzgI8fPhwrVmzRjk5OerRo4eefPJJJSUlacKECaqoqFD37t2VlJTkVobz3XvvvWratKlSUlL0+uuvS5J++ctf6vXXX9evfvWrKvdv166dJk+erAULFqiwsFARERGKj4/X7NmzlZOTo0GDBumll17SwoULtWTJEoWGhiouLk7jx49XUFCQXn31VSUlJWnw4MEKDAxU3759NW3atP84PwCgKj/rYm8QAADgv9S1a1fFx8frzjvv9HUUSdIjjzyili1bOt/jBwCAyTgzCACo9T799FN99dVX2rFjR6XvGwQAwGSUQQBArbd+/Xp98MEHmjp1qktf6wAAgAm4TBQAAAAADMRXSwAAAACAgSiDAAAAAGCgWveewezsbAUHB/s6RhUlJSU1MldNxbzcw7zcw7zcw7zcw7zcw7zcw7zcw7zcw7zcV1NnVlJSonbt2l3yfrWuDAYHBzu/YLgmyc3NrZG5airm5R7m5R7m5R7m5R7m5R7m5R7m5R7m5R7m5b6aOrPc3FyX7sdlogAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDP6MMkeFx7YVHR3tsW15MhcAAAAAMwX4OkBNFuhv07iMlb6OUcWrscN9HQEAAADAZY4zgwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGCgAG/tyOFwKCEhQQcOHFBQUJDmzJmjli1bOtcvX75c69evlyT17NlTEyZMkGVZ6tGjhyIiIiRJ7dq105QpU7wVGQAAAABqLa+VwS1btqi0tFRpaWnKzs5WUlKSFi9eLEk6evSo3n//fa1Zs0Z+fn4aPny4+vXrp7p166pNmzZ65ZVXvBUTAAAAAIzgtctEs7KyFBsbK+mnM3z79u1zrrvqqqu0bNky2Ww2+fv7q7y8XMHBwcrJydHx48c1YsQIjRkzRn/961+9FRcAAAAAajWvnRksLCxUaGio87bNZlN5ebkCAgIUGBiohg0byrIsPfvss2rdurWuvfZanTp1SmPHjtUtt9yizz//XFOnTtXatWt/dj8lJSXKzc31SObo6GiPbKc6eOoYa6ri4uJaf4yexLzcw7zcw7zcw7zcw7zcw7zcw7zcw7zcd7nPzGtlMDQ0VEVFRc7bDodDAQH/2n1JSYmmTZumkJAQzZw5U5LUtm1b2Ww2SVKnTp10/PhxWZYlPz+/i+4nODi4Rpc4T6ntx5ibm1vrj9GTmJd7mJd7mJd7mJd7mJd7mJd7mJd7mJf7aurMXC2oXrtMtEOHDkpPT5ckZWdny263O9dZlqWHHnpIUVFRmjVrlrMAvvzyy3rjjTckSfv371ezZs1+tggCAAAAAFzjtTOD/fv3V2ZmpoYOHSrLspSYmKiUlBS1aNFCDodDn376qUpLS5WRkSFJeuyxxzR27FhNnTpVO3bskM1m09y5c70VFwAAAABqNa+VQX9/f82aNavSssjISOfPe/fuveDjlixZUq25AAAAAMBEfOk8AAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDMJjyhwVHttWdHS0x7blyVwAAABAbRHg6wCoPQL9bRqXsdLXMap4NXa4ryMAAAAANQ5nBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAAd7akcPhUEJCgg4cOKCgoCDNmTNHLVu2dK5fvny51q9fL0nq2bOnJkyYoOLiYk2dOlU//PCDQkJCNG/ePDVs2NBbkQEAAACg1vLamcEtW7aotLRUaWlpmjJlipKSkpzrjh49qvfff1+rVq1SWlqaPvroI+3fv19vv/227Ha7Vq5cqcGDBys5OdlbcQEAAACgVvNaGczKylJsbKwkqV27dtq3b59z3VVXXaVly5bJZrPJ399f5eXlCg4OrvSYHj16aOfOnd6KCwAAAAC1mtcuEy0sLFRoaKjzts1mU3l5uQICAhQYGKiGDRvKsiw9++yzat26ta699loVFhYqLCxMkhQSEqKCgoJL7qekpES5ubkeyRwdHe2R7VQHTx2jJzEv3ykuLq71x+hJzMs9zMs9zMs9zMs9zMs9zMs9zMt9l/vMvFYGQ0NDVVRU5LztcDgUEPCv3ZeUlGjatGkKCQnRzJkzqzymqKhIV1xxxSX3ExwcXKNLiaeYcIyeVNvnlZubW+uP0ZOYl3uYl3uYl3uYl3uYl3uYl3uYl/tq6sxcLaheu0y0Q4cOSk9PlyRlZ2fLbrc711mWpYceekhRUVGaNWuWbDab8zE7duyQJKWnp6tjx47eigsAAAAAtZrXzgz2799fmZmZGjp0qCzLUmJiolJSUtSiRQs5HA59+umnKi0tVUZGhiTpscce07BhwxQfH69hw4YpMDBQCxYs8FZcAAAAAKjVvFYG/f39NWvWrErLIiMjnT/v3bv3go9btGhRteYCAAAAABPxpfMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAZyqQz26dNHCxcuVF5eXnXnAQAAAAB4gUtl8JFHHtG+fft0++2364477tDrr7+u48ePV3c2AAAAAEA1cakMDh48WMuWLVN6erruuusubd68WX379tX999+vtWvXqrCwsLpzAgAAAAA8yK33DDZs2FD33nuvnn32WY0ePVq7d+/WjBkzdPPNN2vGjBnKz8+vrpwAAAAAAA9yuQweP35cKSkpGjJkiAYMGKCdO3fqiSeeUGZmptasWaNDhw7pwQcfrM6sAAAAAAAPCXDlTr/97W+1e/duNW/eXIMGDdKCBQvUsmVL5/oGDRrot7/9rZ5++ulqCwoAAAAA8ByXyqDdbtfjjz+u9u3bX/Q+Xbp00Z/+9CePBQMAAAAAVB+XLhOdOXOm8vLytGnTJueySZMmVSp/DRs2VHh4uOcTAgAAAAA8zqUyuHjxYs2fP18Oh8O5zG63a+7cuVq+fHl1ZQMAAAAAVBOXymBaWpoWLFigW2+91bns4Ycf1rPPPqvU1NRqCwcAAAAAqB4ulcF//OMfatasWZXlLVq00A8//ODxUAAAAACA6uVSGWzXrp2WLl2q8vJy57KKigqlpKTohhtuqLZwQG1W5qjw2Laio6M9ti1P5gIAAEDN5dKnicbHx2vkyJHKyMhQVFSU/Pz89PXXX6uiokJLly6t7oxArRTob9O4jJW+jlHFq7HDfR0BAAAAXuBSGfzlL3+pTZs2acOGDcrLy1NgYKD69u2rgQMHKjQ0tLozAgAAAAA8zKUyKP3ri+UBAAAAAJc/l8rgoUOHNG/ePOXk5Ki8vFyWZVVav3PnzmoJBwAAAACoHi6VwWeeeUYnT57UmDFjuCwUAAAAAGoBl8rg3r17tWLFCrVt27a68wAAAAAAvMClr5a4+uqrVVJSUt1ZAAAAAABe4tKZwUmTJikhIUEPPfSQIiIiFBgYWGl9q1atqiUcAAAAAKB6uFwGJWny5MnOZX5+frIsS35+fsrNza2edAAAAACAauFSGdy6dWt15wAAAAAAeJFLZbB58+aSpKKiIh0+fFitWrVSaWkpnywKAAAAAJcplz5AprS0VAkJCercubOGDBmi48ePa9q0aRo3bpwKCgqqOyMAAAAAwMNcKoMvvviidu/erTfffFPBwcGSpNGjR+vYsWOaO3dutQYEAAAAAHieS2Vw48aNmjFjhjp06OBcFhMTo9mzZ2v79u0u7cjhcOiZZ57RPffcoxEjRujw4cNV7pOfn68BAwY4v8bCsizFxsZqxIgRGjFihBYsWODSvgAAAAAAP8+l9wzm5+ercePGVZaHhoaquLjYpR1t2bJFpaWlSktLU3Z2tpKSkrR48WLn+oyMDC1YsECnTp1yLjty5IjatGmjV155xaV9AAAAAABc49KZwU6dOik1NbXSstLSUiUnJ1c6W/hzsrKyFBsbK0lq166d9u3bVzmIv79SUlJUv35957KcnBwdP35cI0aM0JgxY/TXv/7VpX0BAAAAAH6eS2cGp0+frlGjRikzM1OlpaV66qmndOjQIdlsNr3++usu7aiwsLDSp4/abDaVl5crIOCnCN27d6/ymMaNG2vs2LG65ZZb9Pnnn2vq1Klau3atS/sDAAAAAFycS2UwIiJCGzdu1AcffKBvv/1WFRUVGjhwoAYNGqS6deu6tKPQ0FAVFRU5bzscDmcRvJi2bdvKZrNJ+uns5PHjx51fdH8xJSUlys3NdSnTpURHR3tkO9XBU8foSczLPczLd4qLi2v9MXoS83IP83IP83IP83IP83IP83Lf5T4zl8qgJAUFBemuu+76j3fUoUMHbd++Xbfeequys7Nlt9sv+ZiXX35Z9evX15gxY7R//341a9bsZ4ugJAUHB9foF9meYsIxehLzck9tn1dubm6tP0ZPYl7uYV7uYV7uYV7uYV7uYV7uq6kzc7WgulQG77rrrp8tYe+8884lt9G/f39lZmZq6NChsixLiYmJSklJUYsWLdS3b98LPmbs2LGaOnWqduzYIZvNxtdYAAAAAICHuFQGe/fuXel2eXm5jh49qh07dmjChAku7cjf31+zZs2qtCwyMrLK/bZt2+b8+corr9SSJUtc2j4AAAAAwHUulcGLFb533nlH27Zt0/333+/JTAAAAACAaubSV0tcTLdu3fTxxx97KgsAAAAAwEtcOjP4448/VllWUFCgxYsXX/DL6AEAAAAANZtLZbB9+/YX/ACZ4OBgJSYmejwUAAAAAKB6uVQG33jjjUpl0M/PT4GBgWrVqlWlL5IHAAAAAFweXCqDXbt2re4cAAAAAAAvcqkM9unT55Jf9n7O1q1b/6tAAAAAAIDq51IZ/N3vfqfnn39et912mzp16qSgoCDt3btXq1at0j333KPw8PDqzgkAAAAA8CCXyuDGjRsVHx+voUOHOpfdeuutatOmjd544w09+eST1RYQAAAAAOB5Ln3PYE5OzgXfNxgdHa2vv/7a46EAAAAAANXLpTIYFRWllJQUlZeXO5eVlpbqlVdeUUxMTLWFAwAAAABUD5cuE3366ac1ZswYbd++XXa7XZK0f/9+1a1bV0uXLq3WgAAAAAAAz3OpDN54443atGmTNmwTm+UgAAAgAElEQVTYoLy8PNWpU0dxcXG67bbbFBQUVN0ZAQAAAAAe5lIZlKSGDRvqjjvu0OHDhxUZGamysjKKIAAAAABcplx6z2BpaakSEhLUuXNnDRkyRCdOnNC0adM0btw4FRQUVHdGAAAAAICHuVQGX3zxRe3evVtvvvmmgoODJUmjR4/WsWPHNHfu3GoNCAAAAADwPJfK4MaNGzVjxgx16NDBuSwmJkazZ8/W9u3bqy0cAAAAAKB6uFQG8/Pz1bhx4yrLQ0NDVVxc7PFQAAAAAIDq5VIZ7NSpk1JTUystKy0tVXJycqWzhQAAAACAy4PL3zM4evRoZWZmqrS0VE899ZQOHTokm82m119/vbozAgAAAAA8zKUyGBERoY0bN+rDDz/UN998o4qKCg0cOFCDBg1S3bp1qzsjAAAAAMDDXCqDd9xxh5KSknTnnXdWdx4AAAAAgBe49J7BkydPKjAwsLqzAAAAAAC8xKUzg3fffbfGjx+ve+65R+Hh4apTp06l9T179qyWcAAAAACA6nHRMpiWlqa4uDiFhIRo8eLFkqTnnnuuyv38/PyUm5tbfQkBAAAAAB530TKYmJioHj16KCQkRJKUmZmpRo0aeS0YAAAAAKD6XLQMNmnSRDNnzlRMTIwk6fXXX7/gJ4f6+fnp4Ycfrr6EAAAAAACPu2gZnDt3rpKTk/WXv/xFfn5+yszMlM1mq3I/yiAAAAAAXH4uWgY7derk/EL5Pn36KCUlRQ0aNPBaMAAAAABA9XHp00S3bdtW3TkAAAAAAF7k0vcMAgAAAABqF8ogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABqIMAgAAAICBKIMAAAAAYCDKIAAAAAAYiDIIAAAAAAaiDAIAAACAgSiDAAAAAGAgyiAAAAAAGIgyCAAAAAAGogwCAAAAgIEogwAAAABgIMoggMtCmaPCY9uKjo722LY8mQsAAMCbAnwdAABcEehv07iMlb6OUcWrscN9HQEAAOA/wplBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAXiuDDodDzzzzjO655x6NGDFChw8frnKf/Px8DRgwQCUlJZKk4uJiTZw4UcOHD9eYMWOUn5/vrbgAAAAAUKt5rQxu2bJFpaWlSktL05QpU5SUlFRpfUZGhh544AGdOnXKueztt9+W3W7XypUrNXjwYCUnJ3srLgAAAADUal4rg1lZWYqNjZUktWvXTvv27ascxN9fKSkpql+//gUf06NHD+3cudNbcQEAAACgVgvw1o4KCwsVGhrqvG2z2VReXq6AgJ8idO/e/YKPCQsLkySFhISooKDgkvspKSlRbm6uRzJHR0d7ZDvVwVPH6EnMyz3Myz3My3eKi4tr/TF6EvNyD/NyD/NyD/NyD/Ny3+U+M6+VwdDQUBUVFTlvOxwOZxF05TFFRUW64oorLrmf4ODgGv2i0VNMOEZPYl7uYV7uqe3zys3NrfXH6EnMyz3Myz3Myz3Myz3My301dWauFlSvXSbaoUMHpaenS5Kys7Nlt9tdesyOHTskSenp6erYsWO1ZgQAAAAAU3jtzGD//v2VmZmpoUOHyrIsJSYmKiUlRS1atFDfvn0v+Jhhw4YpPj5ew4YNU2BgoBYsWOCtuAAAAABQq3mtDPr7+2vWrFmVlkVGRla537Zt25w/161bV4sWLar2bAAAAABgGr50HgAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEABqoTJHhce2FR0d7bFteTIXAAD47wT4OgAAwPMC/W0al7HS1zGqeDV2uK8jAACA/8eZQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAGC8MkeFx7YVHR3tsW15MhcAAP8uwNcBAADwtUB/m8ZlrPR1jCpejR3u6wgAgFrMa2XQ4XAoISFBBw4cUFBQkObMmaOWLVs6169evVqrVq1SQECAHnzwQfXu3VtnzpzRr3/9a9ntdklSv379NHLkSG9FBgAAAIBay2tlcMuWLSotLVVaWpqys7OVlJSkxYsXS5JOnjypFStWaO3atSopKdHw4cPVvXt3ffXVV4qLi9OMGTO8FRMAAAAAjOC19wxmZWUpNjZWktSuXTvt27fPuW7Pnj1q3769goKCFBYWphYtWmj//v3at2+fcnJydO+99+qRRx7RiRMnvBUXAAAAAGo1r50ZLCwsVGhoqPO2zWZTeXm5AgICVFhYqLCwMOe6kJAQFRYW6rrrrlPbtm1100036f3339ecOXO0aNGin91PSUmJcnNzPZLZkx8C4GmeOkZPYl7uYV7uYV7uYV7uYV6+U1xcXOuP0ZOYl3uYl3uYl/su95l5rQyGhoaqqKjIedvhcCggIOCC64qKihQWFqaYmBjVrVtXktS/f/9LFkFJCg4OrtFP6p5iwjF6EvNyD/NyD/NyD/NyT22fV25ubq0/Rk9iXu5hXu5hXu6rqTNztaB67TLRDh06KD09XZKUnZ3t/FAYSYqJiVFWVpZKSkpUUFCgvLw82e12TZ8+XX/+858lSTt37lSbNm28FRcAAAAAajWvnRns37+/MjMzNXToUFmWpcTERKWkpKhFixbq27evRowYoeHDh8uyLE2ePFnBwcGaMmWKpk2bprffflt169bVnDlzvBUXAAAAAGo1r5VBf39/zZo1q9KyyMhI589333237r777krrw8PDtWLFCq/kAwAAAACTeO0yUQAAAABAzUEZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAC4pcxR4bFtefL7uTyZCwBM4LVPEwUAALVDoL9N4zJW+jpGFa/GDvd1BAC4rHBmEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAAAAwEGUQAAAAAAxEGQQAAAAAA1EGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAAADAQZRAAAAAADEQZBAAAAAADUQYBAAAAwECUQQAAgGpU5qjw2Laio6M9ti1P5gJweQrwdQAAAIDaLNDfpnEZK30do4pXY4f7OgIAH+PMIAAAAAAYiDIIAACAGoPLagHv4TJRAAAA1BhcVgt4D2cGAQAAAMBAlEEAAAAAMBBlEAAAAAAMRBkEAAAAAANRBgEAAADAQJRBAAAA4DLFV3Hgv8FXSwAAAACXKb6KA/8NzgwCAAAAgIEogwAAAABgIMogAAAAABiIMggAAAAABvLaB8g4HA4lJCTowIEDCgoK0pw5c9SyZUvn+tWrV2vVqlUKCAjQgw8+qN69eys/P1+PP/64iouL1aRJE82dO1d169b1VmQAAAAAqLW8dmZwy5YtKi0tVVpamqZMmaKkpCTnupMnT2rFihVatWqVXnvtNT3//PMqLS1VcnKy4uLitHLlSrVu3VppaWneigsAAACgFuLrOP7Fa2cGs7KyFBsbK0lq166d9u3b51y3Z88etW/fXkFBQQoKClKLFi20f/9+ZWVlady4cZKkHj166Pnnn9f999/vrcgAAAAAahm+juNfvFYGCwsLFRoa6rxts9lUXl6ugIAAFRYWKiwszLkuJCREhYWFlZaHhISooKDgkvspKSlRbm6ux3I/+ov2HtuWp3jy+DyNebmHebmHebmHebmHebmHebmHebmHebmHebmvts+spKTEpft5rQyGhoaqqKjIedvhcCggIOCC64qKihQWFuZcXqdOHRUVFemKK6645H7atWvn+fAAAAAAUMt47T2DHTp0UHp6uiQpOztbdrvduS4mJkZZWVkqKSlRQUGB8vLyZLfb1aFDB+3YsUOSlJ6ero4dO3orLgAAAADUan6WZVne2NG5TxP9+uuvZVmWEhMTlZ6erhYtWqhv375avXq10tLSZFmWxo0bp1//+tc6deqU4uPjVVRUpAYNGmjBggWqV6+eN+ICAAAAQK3mtTIIAAAAAKg5+NJ5AAAAADAQZRAAAAAADEQZBC4DXM3tvuPHj/s6AgAAQI1GGfQQXqy7h3m5zuFwyM/PT5J09OhRH6ep+crLy7Vs2TKtW7eOQugih8Ph6wiXDWYF1Czn/kzyusJ1aWlpKi4u9nWMy0JFRUWVZbXtd82WkJCQ4OsQlzuHwyF/f3+Vl5fL3/+nfm1ZlvMFPCqrqKhwzulc0WFeF3bud0uStmzZokWLFikoKEjXX3+9j5PVTJZlyWazKSgoSFlZWSouLlZ4eLiCgoJ8Ha3GsixL/v7+OnHihLZt26aCggI1a9bM17FqpPP/PG7dulVHjx5VUFCQwsLCfJys5jr/7/Yff/zR+WcUF8fzoXv8/PyUk5Oj1atXq27dumratKmvI9Vo5eXlmj9/vk6cOKHOnTvzu/Yzzj0/5ufnKzs7Wz/88IOuuuqqWjczyqAH+Pn5affu3UpOTtbu3bt10003UXB+xrkXnvPnz9fu3bsVEhKiq666ytexaqRzvz/PPPOMcnNz1aRJE33zzTcKDQ1VeHi4j9PVPOfmdfjwYW3YsEEHDx7UlVdeqfDwcF6AXoSfn5/27NmjWbNm6eqrr9bmzZv19ddfq2vXrr6OVuOc+/168803lZqaqrKyMq1du1a9e/dWcHCwj9PVPOc/B27atEkLFy7Uxx9/rG7dujGvCzg3Lz8/P+3cuVPvvvuuCgsLdd111/k6Wo1z/hUzO3fu1IIFC9SvXz+VlZUpODhYoaGhPk5YM5WXlysgIEB2u10vvfSSgoKC1LZtW1/HqrH8/Py0f/9+PfXUU2rUqJGWLFkiSWrTpo2Pk3kWl4l6wMcff6xXXnlFPXv21I4dO5SUlCRJFMGL+P777xUfH68+ffooNDRUv//97/Xtt99Kqn2n3v9T588hLy9PZ86c0XPPPadJkyapY8eOevfdd3Xo0CHfBaxhzr90b8+ePZozZ46efvpp3XzzzcrJyVFWVpYP09U8/36p4//+7/9q1KhR6tKli06dOqWzZ8/qwIEDPkpX85z/53HVqlX69NNPtWLFCk2fPl1t27ZVYmKiD9PVXOeeAzdv3qz169fr4Ycf1hVXXKEJEyb4OFnNdG5eH3/8sZYsWSLLsrR161a9+eabPk5Ws5x/hv706dPaunWroqKiFBoaqhUrVujpp5/WypUrfZyyZgoICNCuXbv0wgsvqG/fvlqyZIkyMzN9HatG+ffnx1WrVmnkyJEaMGCAAgIC9Nlnn+nIkSM+Slc9KIP/gfOvH/7uu++0fPlyXX/99RowYIBSU1O1Y8cOrVixwocJa5bz/2A5HA4VFRWpRYsWat68ubKyshQbG+t8sW56gT73otPPz09nz551XtP/3XffSZIaNGiga665RqdPn9bbb7+twsJCn2WtKc5/YbBr1y7l5eWpU6dO+uUvf6mRI0eqtLRU7777rvbu3evjpDXD+ZdpZ2dn6+zZs6qoqNDWrVv1/PPPa/bs2YqKinL+zpnu3NmakpISST/92fT399fnn38uSRo1apT++c9/8uLzPOc/R+bl5Wn9+vVq2rSpYmJiNH36dNWvX18PPfSQDxPWLOc/R27evFmpqam6//779eijj2rQoEHav3+/3n//fR8mrFnO/f01Z84cvfXWW4qLi9OVV16pzMxMLV26VLGxsVy6fZ7zf7+Kioq0ceNG3XnnnXriiScUHx+vBQsWKCcnx4cJa47znx/z8vJUUlKili1bateuXXr88ce1YMEC9erVSwcPHvRxUs/iMlE3VVRUOC83O3LkiIKDg9WwYUNt375dzZo1U2RkpNq0aaM//vGPGjRokAIDA40uOOfPa8uWLdq4caPCw8O1a9cuLV++XAsXLlSzZs20fv16derUSXXr1vVxYt8697uyfft2JSYmKjMzU3FxcTpy5IhSU1PVvHlzvfbaa2rdurUKCwvVqVMn1atXz8epfcvPz09lZWWaMmWKAgIC1LdvXy1evFiNGjVSq1at9Mknn6isrExxcXFcmqZ/vZBKSEjQ+++/r8aNG6tBgwZKS0vT+PHjdfjwYa1atUp33XWXGjZs6OO0vufn56fs7GzNnj1bubm5GjJkiM6cOaPvv/9ewcHBatmypX71q1+pS5cuvo5aIzgcDtlsNp09e1Yff/yxIiMjFRQUpIMHD8qyLF133XXq16+fMjIy1L17d+Pfz3vuH7Msy9KZM2fUpEkTffbZZ7LZbLLb7QoPD1dRUZH279+vzp07KyAgwNeRfeb8y47feustffLJJxo1apTatGmjqKgonTx5UuvXr9euXbs0duxY419PnHNuZi+88IL8/f1VUFCgI0eOKDY2VpGRkdqwYYO++eYb9erVy/i3U5x7fpw1a5bee+89BQUFqbi4WOnp6Ro2bJhOnjyppUuXasiQIbXq+ZEy6CZ/f3+dPn1a06dP144dO5STkyN/f3/dfPPNWrdunVq1aqXWrVvrrrvuUt26dY0uglLlf8H76KOPdOrUKUnSVVddpTp16igvL08rV67UxIkTFRER4cOkvnXgwAGFhYUpICBAO3bs0DvvvKMJEyaooKBAr776qhYuXKiioiJ9+eWXiouLU69evbRy5Ur16NFD9evX93V8rzv/DKokffTRR1q9erVGjhypyMhIXX311Zo/f74+//xznTlzRomJiUa/MPjhhx+0fPlyde7cWZK0dOlSnThxQi+//LJatGihyMhIHT58WCEhIcrKytK8efP4EJn/t3PnTr388suaNGmS1q5dqzNnzmjw4MH64osvdPToUUVFRalBgwa+jllj+Pn5KS8vT0899ZQ+++wzHTt2TB07dlRgYKCys7MVEBCgiIgIDRgwwPgiKP00r7/97W+aPHmy/vSnP6lJkyYaOHCgNmzYoMDAQEVEROi6665TbGysAgMDfR3XZ86dsfnhhx/k5+cnh8OhgoICFRYWKjw8XKGhoSooKNCpU6eUlJSkevXqVXpfoYnOL8+vvfaa0tPTNWnSJP3iF79Qdna2MjIylJubq6KiIj366KO1qty449ylxpGRkfL399fLL7+sEydO6IUXXlDr1q0VHh6ugwcPqn79+vroo480f/58XXPNNb6O7VGUQRccOHBAn376qa6//nqVlZVp2rRp6tixoyZOnKj69etrw4YNuvHGG1VUVKSioiK1adNG/v7+Rv8ldL7U1FQdP35cCxculCQdO3ZMjRo10v/8z/+orKxM48ePV0REhLF/cR86dEj79u1T27Zt9fXXX2vdunXy9/fXvffeqy5duigrK0tvvfWWpk+frvLycv3tb3/TnDlz9OCDD6pDhw6+ju9153/Iwp49e3T69Gk1bdpUDRo00KZNmxQTE6OYmBgNGDBAN910k+6++27j/7WzXr16CgsLU3BwsIKDg5WTk+P8hyubzaZVq1apW7du+vWvf61+/fqpTp06vo7sM+dfJiT99KmhN910k66++mrl5eXpz3/+s4KDg9WxY0fFxMSoefPmPkxbM5z/ojM/P18LFy7UI488ogEDBmjx4sWqV6+ebrjhBhX+H3vnHRXl1e7tawp9pFfpHWkiKkoQKXbUYI31TWLsR2NiNJqob4wt1tgbxmCJ2HtBxW4kImIBQURUELAgRRCQznx/eJgMJudt5zsZk2eutVwrKLPWb3Z2ue+971JejoGBgaAv/t4mOzubBQsWMGbMGPz9/ZkyZQr9+/fH3t6effv24e3tjZmZmeL1UGhnZG5uLgYGBojFYs6cOcOKFStITk5GS0sLPT09iouLKS0txcXFBUdHR9q1a4dIJGoSlSRElMPbpVIpt27dor6+HisrK9zc3HBycuLFixdUV1czdepUTE1NBWuDVVVVIZVKMTQ0RCqV8ujRI1xcXPDy8kIkEnH69GlatmxJjx496Nat218yGkvtDP4T6urqqK6uxtbWlpqaGmQyGfv37+fTTz9FX18fU1NTUlNTCQ0NJTg4GD8/P0C4uW/Kh1XjxnL+/HksLCzw9/fHzc2NkydPkpeXh5ubG0FBQejo6DTJ+xIKcrmcxMREXFxc8Pb25vDhw9TV1WFra8vTp08pLy/H3d2dTp06kZiYSIcOHTAyMqKmpoa+ffsqXnmERuP82rdvHxs2bOD+/ftUV1fTunVr6urqOHHiBCEhIRgYGKjzRpSQSCQMGzYMsViMu7s7K1asoE2bNhQUFLB582a8vLwEX7VQ2YBMS0tDS0uLFi1acP36deLi4li4cCEFBQXcuXOH4cOHq19P4Td7t0gkIjs7G5FIxKFDh+jTpw9bt27FzMyMrl274uPjo0K1qudtg7uoqIiqqiqsra25fPkyjo6OzJs3j9GjR+Pn54ebm5vid4VmV/zwww/ExcURHh5Obm4u69atY/369WRmZnLp0iXCw8Opq6sjIyMDDw+PJvu90OyJtxGJRCQlJfHdd9/x5MkT+vTpw5MnT8jLy0NfXx9nZ2f8/f0JCAhAQ0ND0M6zjo4OtbW1TJ06FUNDQyoqKkhISMDOzo7S0lJWr15N69atsbe3V7XU/zPUzuA/4MmTJ+Tk5ODm5kZtbS0ffvghwcHBVFRU8NNPP9G7d29u3brFgQMHCA0NFXx7BGVH8OzZs8TGxvL69Ws6dOjA6tWrcXBwQE9PjyNHjlBXVweAn5+fIB1BeLNZ3717l3Xr1lFWVkZGRgZPnz7F09OTZs2akZaWRmVlJc7OznTq1AmpVIqOjg52dnaYmJioWv4fjrIRdfPmTY4fP050dDQ5OTnEx8djaWmJq6srJSUlNG/eXJDhs8oor8dr164RGxvLZ599xvz584mMjMTOzo7z589z+PBh/va3v9G1a1cVK1Y9YrGYkpISvvrqK5KSkhR9F8PDw/nll1+4evUq+fn5fPfdd4KfX9DUeV67di0ZGRkUFRUxePBgjh07Rp8+fTAyMuLGjRt8+OGHgn8RVB6v2NhYamtrAXB1deXHH39k9OjRODs7Ex8fT1hYmKJ8vRBfBKurq0lMTMTf35+qqipKSkpITk6mvr6eGzduMGDAAF6+fElkZCSenp6Ct7/e5tKlS0RFRTF+/Hiio6PR1tama9euXL9+nRcvXuDm5qbIoW/spScklNdUSkoKSUlJBAQEsGfPHvr3709hYSFpaWns3LmTYcOG0aVLFxUr/r9F7Qz+AyorK5k/fz6HDh3Czs4OR0dHfvzxRyZOnEhKSgpHjhzh9OnT/P3vf8fT01PVclWOcmL3sWPHaN++PVu2bMHJyYmQkBC2bt3K0aNHmTZtGmZmZqSkpBAeHi64Q07ZqcnMzOTgwYOIRCLmzp1LUlISL168wNXVlerqagwMDAT/WgO/GlFyuZzKykqqq6sRiUQkJydTUVGBqakp58+fx9LSkiFDhmBqaqpqySrlbePRwsKCtWvXEhoaSsuWLfn2228ZPXo0vXv3plOnTuo+U/9NQ0MDc+fOxd3dXdE2YuXKlRgZGWFsbExZWRkLFy4UdP4pwOvXr9HQ0EAsFlNTU8NXX31FRUUFbdu2Zc6cOZiZmdGsWTPmzJlDSkoKCxcuxNHRUdWyVUZj6HGjwT1jxgxFA+tTp07h5eVFRUUF+/fv5/LlyyxZsgR3d3fF54V2RsKbFgi5ubksWrSI58+fM2LECEWLkt27d3P16lUuXrxI586d1Tm7/Da8/dSpU0RERCh6Ex8+fBhbW1vc3d3x8/Nr4jwLbX69fT5WVFSwePFiJkyYQH19PSdOnGDYsGH07NmT4OBgWrVqpUK1fwxqZ/B3aDTWZTIZubm5nD9/nkGDBhEWFkZOTg4nT55k8eLFhIWF8f7772NnZ6dqySpFeRN69eoVx48fZ/r06ZSXl/PkyROuXLlCWFgYo0aNUhj169ev529/+5sgx065Yujp06cZPnw4xcXF5OfnM3LkSPbt24e2tjZdunTB19dXxWrfDcRiMU+ePGHWrFkkJiYSFBSErq4uiYmJTJ06lStXrmBkZMTHH38s+IqhjS/t1dXVLFq0iIKCAry9vZFKpVRWVhIeHs7z5885c1PpLg4AACAASURBVOYM3bp1E3R+4NvU19dz4cIFhgwZgoGBAQYGBjx//pw2bdrQrVs3OnbsqGqJKkUul9PQ0MDHH39MdXU1vr6+ZGdnc/36dZYsWYK9vT0+Pj6sXr2aL774Ai8vL/7rv/5L0MZ6ZWUlU6ZMwcbGBgsLC+Li4sjNzWX58uX4+vpSUlLC7du3ee+995DL5UybNg0rKyvB5m81Gup1dXVUVVVRVVWFqakpNjY2BAQEkJmZSWJiIvfv32flypXq/YtfK/gCHD58mMrKStq1a8fFixeJj49n0aJFPHjwgAcPHvDhhx9iYWGhYsWqo/F8rK2t5ZtvvsHY2BhPT090dXV58uQJ/fv35/z58+Tm5hIYGIienp6qJf8hqJ3B30EkEpGVlcXx48dp1aoVnp6eHDp0iMDAQNq2bcvFixcpLCykVatWgq+G1rgJVVdXc+HCBfT19TEyMuLs2bM8ffqUMWPGcOXKFVJSUoiIiODZs2fcuHGDMWPG0Lp1a1XL/0NRvo3asWMHW7duZcGCBfj7+6Ojo0NcXBz379/H2tqadu3aKRxlIYYIvc2rV6+YPHkyERERdOrUCUdHR2pra1m7di379u3DyMiI2bNnCzbnQRmRSMSDBw/YuHEjjo6OxMXFUVxczLlz5ygsLCQ8PJzAwEC6deumaqkqZceOHbx8+bJJ8SqxWMytW7fYvn07ffr0ISEhgV27dhEaGipoA6qRxjFycnJi+fLleHh4YGBgQFxcHN27d0cikVBbW0tmZiY9e/bE3t5e8GtSQ0ODwsJCRfXnxoJhERERaGpqkp2dTX19PX369KFt27Zoa2sLOn9LJBJx+fJlYmJisLS0ZOTIkSQmJpKRkUHr1q0ZMGAAXl5eDBo0CIlE8psXMSEiEol49eoVn3zyCVVVVRQUFNCpUyfq6+u5e/cuCQkJlJWVMX/+fGQymarlqhSRSEROTg4HDhygpqaGmzdvkpyczKNHj9DQ0MDf35/OnTsrChEJBbUz+Dtcu3aNL774Ajs7O0QiEf379+fGjRucPHmSs2fP8uWXX/Lee++pWuY7QWMZ8blz57J7925sbGxo164djx8/xt/fn127duHs7MysWbMURkT79u0xNzdXtfQ/lLfzIjU1Ndm/fz9BQUFYWVlhYGCAubk5cXFxjB49usmLqZA2JGWUneDCwkJ++eUXpk2bhqmpKWlpacTFxbFgwQK8vb0ZPny4itWqFuWxevr0KdOmTcPX15cRI0bg7++vyNt6+PAhfn5+mJmZqVix6nn06BFbtmyhbdu2GBkZUVtbi0QiISgoiOvXr3P16lVOnDjB3//+d/ULvRJ1dXVYW1tjZGTEsmXLGDp0KBkZGRw/fpzCwkJ++OEHOnToQMuWLVUt9Z3B19eXx48fc+LECcaMGcOJEyfIysrC1NSUqKgoWrZsqcgPBGEXP7l8+TI//PADY8eO5cWLF2hraxMWFsaJEyd49uwZfn5+inxd5RcxIaL8enz69Gnq6uqYM2cO7du35/Lly4p2QRUVFXz33XdoaWkJ0nlWPh+fPXvGJ598QnBwMGPHjsXNzQ09PT0OHz7M7du3adeuHSYmJoIbI7UzSNMFVVtby8GDB+nfvz+DBw/GycmJY8eOMWDAAEQiEe7u7oIs569MeXk5EokEsVhMbm4u06dPZ8yYMQQGBnLu3DlMTEx48OABCQkJyGQypk6dilgspq6uTpAtNxqTs58/f87f//534uPjad68OW3atGHp0qV069YNQ0NDrK2t6dmzJzKZTNCvgRkZGejp6TXpqaWvr8+hQ4e4f/8+QUFBpKSkEB8fT9euXf/SFb7+FZQP9/r6empraykuLubevXuEhoZiamqKhYUFbdu2pV27dk2MTiHj4eGhKAbWpUuXJuFmDg4O9OjRg8jIyL9cP6n/hPT0dF6/fo2hoaFirrm5uVFYWMj27dtZuHAhWlpaFBQUEBERQUREhIoVq5a3QzxFIhEtW7bkypUrJCcnM23aNE6cOMH169fp1asXffv2VaFa1fK2c3Lw4EEqKyuxsLDgyJEjHDt2DIlEwoABAxShto0I9Yy8ffs2lpaWin6LIpGI4uJiLl26REBAADKZjGvXrlFfX8+IESMU4e1CfHF++yJeIpFw584dcnJy6N69OwYGBtjb2xMQEEB4eDienp6CnFeCdwaVF0dlZSVaWlrcuXOHhIQEunbtikQiYcOGDTg6OhIWFoarq6uKFauW169fc/r0aZo1a0ZtbS21tbXEx8czceJEnJ2defHiBfv372f48OH079+fzp07A8K+wWsM25s5cyY9evQgICCABQsWMHDgQPT09Fi0aBFDhw5V/K5QHcHG733nzh12795NVVWVonS/RCIhMDCQtWvXcufOHU6fPs3MmTPVoXv8+oqwfPlyLl68SG5uLv7+/tTX13Pt2jUCAwMBkMlkgh6v31tX3t7eiletxrDZUaNGUVZWRocOHQTd5FuZ58+fs2HDBp48eYKdnR2ampqIxWLat29PfHw8+/fv5/PPP8ff3x9bW1tVy1U5jfMsPT0dMzMzGhoa0NTUxM/PjwMHDlBeXs60adMICwvDw8MDEGY6QONFaX19PUePHkVLS4sOHTooQh1nz55NaWkpIpGIoKAgQVbRfpuCggK++eYbqqqq8PX1VcwZDQ0NXr58yfHjx6mqqmLLli0MGDBAcYYKsWIo/LoWV6xYQWJiInfu3FGEHj98+FDRosvY2FjQ56PgncHGamiTJ0/m7t27bN++nb59+/L48WPOnTtHWloajx8/ZvDgwYJPVK6vr0dLS4vk5GQmTZrE3bt36d27N6mpqTx9+hRfX1+eP3/OxYsXyczMVNx2CrF1xNs3ww8fPqS4uJixY8fSvHlzmjVrxrZt2/j222+xsrJqUmlPaAYBvLlkWL9+PT4+PlRVVbFy5UqKi4vp168fEomEuro6mjVrxvvvv4+vry9Dhw4V9MatjFwu5+uvv0YmkxEREcHBgweprq6mY8eOXL58maqqKoXBKVSUDe3c3FzEYjFaWlqIxWL8/PyIj4/n1KlTHDhwAB8fHyZNmqRixe8WVVVVbNy4kWfPnjFo0CA0NTWpqalBIpHQpUsX0tPTadeunWAv/BpR3vevXr3K8uXL6devn+LvdHV1cXZ25vLly4rLhsZ/E+K+LxKJePnyJdOnTyctLU3RTmngwIFcu3aNvXv38vDhQ6ZMmYJUKlW1XJVTX1+PTCbDycmJ7du3Y2Njg7W1NfDmss/BwYHa2lpycnIYN26cou81CHN+yeVyAGbNmoVUKqVv3778+OOP1NXVMXToUFatWoWFhYW6YjsCdQYfPnxITk4OlpaWirLY7dq1Y+TIkfzwww9UVFQwceJECgoKEIvFzJw5UzAVhf4nGl/2GhoaqK2tpaKiAplMRkhICDKZjNjYWOLi4rh37x5RUVGcPHkSb29vjIyMBLcJKTu/V65cQUtLi+zsbJ4+fYqzszP6+vpkZWVRWlpKWFiYwhEU4s0wQFJSEvr6+rz33nvU1tbS0NCAgYEBAPn5+Xh7eyvGU1NTE319fUG/2Lw9T0QiEbdv3+ajjz5i06ZN2NnZ4evri62tLa1atSIwMFCQ80qZxu+/b98+YmJiCA0NVbSH0NTUxMPDg23bttGzZ09Gjx6tSqnvBMpz7OXLl1haWuLg4ICOjg7x8fEEBQU1cfw6dOggeEcQ3syz58+fI5PJsLW15c6dO7x69apJmwgzMzPCw8ORSCSCXJfKcysnJ4f169fj6enJ7Nmzkclk/Pzzz9TV1REeHg7A9OnTkUqlgq2uqoxYLKasrIyffvqJ2tparly5gre3NyYmJsjlcmQyGd7e3rRr1w4LCwvBjlnjHGv8k5KSwqBBg4iOjsbFxQVra2scHR0JDg7Gz89PvXchQGewtLRUUZK4devWmJqa8ujRI8zNzVm1ahXDhg3D0NAQY2NjQkND8ff3V08UUMSkjx07FisrKz777DPOnDnD48eP6d69O6GhoRgYGNCxY0fOnTtHfn4+AwYMEORtXuPmO3/+fE6ePElycjK6urrcu3ePnJwcjh8/TmJiIuPGjWvSD0+Imza8yRFZtGgRw4cPZ9GiRdy9e5dRo0ahr6/P+fPnyc/Pp76+HisrK1VLVTlvN8pNTU3FycmJ6OhoVqxYwahRo4iMjGThwoX4+Pjg7e0t2HkFTV9qUlJSiI6OJjg4mPbt2zf5PX19fSIjIwVX4fj3UJ5jR44c4bvvvuPq1asMGTKE5s2bc+PGDS5dusTr169xdnYW9PyCpnOsqqqKuXPnEhcXh4uLCw0NDejr6+Pk5PQbw1yIl3/KF6U1NTXU1tZy6tQp9PX18fb2pnnz5lRXV3Py5Em6dOmi6O8mxFy3Ru7du0d2djZWVlZUV1fzxRdf4OLiwhdffEFDQwO7d++mY8eOiiq0ynUZhDa/lJ3A9PR00tPTad68OVu3bmXv3r107dqVIUOGsG7dOjw9PfHy8hLsvHobwcTuyeVysrOzCQ8Px8vLi8jISFatWkVeXh75+flERUXRs2dP2rRpw549eygrK1O1ZJXT0NDQ5OfLly9TW1uriEH/6quvSEtLY8iQIVy9ehUnJyf2799PVlYW69atE2RYrVwup6qqijFjxtCsWTNiYmLo27cvZWVlDBw4kL59+9K+fXtWr16Nh4fHb8ZYSNTV1QEwceJEmjdvzsKFC/nmm28oKCjg4MGDeHh40L17d/bt20d1dbWK1b4bNB7ue/fuZf78+RQXF5Oens7KlSsxNjbmyZMnjB49mp49ewresVHuJ5Wbm4uxsTE9evTg0aNHpKenA7+GEQGCj/5opHGO7dy5k4SEBFavXo1IJGL16tVYWFgwePBgnj9/jpGRkeDC/99GufjJzz//THp6Ot999x1ubm4cO3aMH374gT179vDy5cvfjJUQDfXGMVi/fj3Tpk0jIyODr7/+muzsbBITExGLxYSFhTF9+nRFxVBA0AZ7TU0Ntra2vH79Gm1tbdzc3BQRWQMHDkQulzN37lxA2OMEv66p/fv3M2/ePO7fv09hYSGTJ09GKpUikUgYP348gYGB6grRbyGYl0G5XI6RkRFSqZQVK1bw7bffkpKSwoMHD+jVqxdPnjxBJBIRHR3N+PHjadeunaolq5zGhXXlyhVKS0vR1dXF3NyclJQUzM3NFW0kAgMDCQoKwsjICH9/f7p3765i5X8syje8IpEIqVTKiRMnePz4Mf369cPOzo4zZ85gYGBAUFAQHh4eaGpqCvq2E5oWPtHT0yMxMZFXr14xadIkoqKi0NTUJCIigt69e6tj+pXIy8sjKiqK1atX8/TpU44fP05+fj5Tp07F3NycwMBAOnXqpGqZKkckEpGdnc3MmTO5d+8eeXl5GBgYYGhoyK1bt3B0dKRZs2aqlvnOoOzYvHr1itWrV9OsWTP69OlDWFgYe/bsISsri969e9OtWzfBV/GFX/ewZcuWcerUKR48eMDjx48ZP348AQEBlJaWUlBQgKWlpaDHS/mMXLRoERUVFURERDBp0iQ++OADnJ2diYmJwdbWFnt7e/T19VWsWPU0rkcLCwuKi4v54osvaNmyJSkpKeTl5WFnZ0d6ejo5OTlERETg4OCgasnvBE+fPiU6OprFixdTVlbG4cOH0dfXp1u3bhgZGdG6dWt69eqlapnvHIJwBpVDE/z9/cnMzGTXrl0sXryY3bt3o6mpybBhw2jZsiWdOnXC29tbxYpVS1xcHPb29ojFYhYvXszZs2fJyckhNzcXqVSKqakpV69exdPTE1NTU0XPwPr6ekUejpBoPOSOHz9ObGwsmZmZzJw5kxMnTvDw4UNcXFzYsWMH7dq1a+LUCPlWvTFkaufOndy8eZOlS5fSp08fli1bhqWlJREREcTFxSnCX4TM2+Fk+vr67Ny5k9LSUh4/fszkyZNZsWIFYWFhuLi4CLqwjvJY1dTUMH/+fD755BPc3d3ZvXs3zs7OODg48PjxY2QyWZN+nkKm8dWmuLiYxMRERCIR4eHh7N+/Hx0dHVq0aEHLli1JSEggKChI0Dm75eXlrF27Fm9vb7S0tNi5cycvXrxgxYoV5OXlceHCBWpqavD19SUgIICnT58ikUgEW8SpcU1WVVVRUlLCtWvXmDBhgqJdyZo1axg5ciRGRka0bdtWkKklb6Ncff3Bgwc4OjpSWFhIQkICAwcOJDExkVu3bnHkyBEmTJhAQECAihWrhurqaioqKprYCLW1tezdu5eamhru3r1L//79iYqKYuDAgbi6uqpbBf0P/OWdQeUeb9u2bePZs2eMHDmSS5cucfPmTb7++mvmzp2Lq6srHh4eyGQyVUtWKRs3bmT+/Pno6+ujpaXFzz//zPr162ndujVlZWUUFxfj6enJ69ev8fb2brIIhebcKOeA7Nixg3PnzjFq1ChWrVpFaWkpU6dOZcaMGZw/f56ZM2fy3nvvqVixalE21Kurq5FKpZSXl1NXV4erq6tizs2aNYuxY8fSu3dvwc2pt2kcs+LiYmpra9HS0gKgT58+aGlp4eLiwtGjR9HT0+P9998X9Hi9nY8kkUg4cuQIFhYW7Nq1i6FDhyISifDz86N169aCNc5/D5FIRFpaGrNnz0ZPT4+jR48iFovp168fq1evxt7eHk9PT0JDQwUdzQBvCg7FxMTw8OFD2rdvj1wux9zcnOjoaGxsbHB0dGTt2rW0atWK5s2bc/DgQaqqqhQtXoSCcl/hBw8eMGHCBIKDgzEzM+Onn37C2tqaL7/8kpiYGMzNzenTpw9SqVSQuZRvIxKJeP36NZ9++qmiquq8efO4fPkyT58+ZdSoUXTv3p1u3bphZ2cn2DFbsWIF58+fp127dmhqaiKXy9HV1aVbt25oamoSHh7OhQsXEIlEdO/eXfB71z/iL+8MNlYSmjNnDo6OjmRmZrJ9+3Y2bNjAsmXLAPjyyy/VhsF/Y2Vlxe3bt6moqODatWsUFBTQv39/tLW1KSoqIjU1lQ8//JBWrVoJ8hWwEeUQT7lcTmJiIhEREaSlpfHq1StkMhleXl506tSJ/fv388EHH2BsbPybBrtCovGwunTpElFRUVRVVZGdnU1JSQklJSWUl5dz6dIlBg4cKNibzrcRiUTcvHmT1atXY21t3aTvYkVFBUeOHEEikTBr1izBzitomo+0du1afvnlF+zs7Kivr+ebb75h48aNSCQS1qxZQ0hIiPp2GDh79iwbNmyga9euAGzYsIGBAwfi5eVFXFwcEomE9u3bo6+vT2VlpfqM5NcLQH19fTZt2kRDQwPBwcFUVVVRVVVFt27dOHXqlCKUFkBbW5tBgwapWPkfS0lJCRcuXMDNzY1Hjx6xatUqfHx86NmzJ3Z2dmRkZFBSUsKPP/5Iv379FH12QXi5lG8jl8upqKjg66+/pnPnzkyZMoVt27aRmZnJrFmzWLduHSYmJri4uKClpSXo1l1+fn6cOHGCgoICfH19FTaZtrY2Dx484MiRI5SUlDBv3jy1I/hP+Es6g29X7Tpw4AAeHh588sknhIaGEhsbS21tLePGjSM3N1fwhufr168VYT91dXU8ffqU9u3bU1VVRWZmJo8ePaJFixasXr0aBwcHAgICBL9hi8ViysvLmTt3LgUFBeTn53P06FGqqqpYsmQJly9fpqGhgcDAQDQ0NNi8eTORkZFNKn0JBeX1ePLkSaKjoxkzZgyxsbFIpVIsLCyora3lxIkThIWFKfpTCpXG8WpoaKCsrIzhw4fz3nvv0a9fvyaXCSYmJrRp04aQkBAVK1YtysbQjh07uHv3Lq6urixatIhJkyYhkUi4fv06J06cYOHChTg7O6tY8buBk5MTK1asoL6+XtFrMS8vjyNHjiguF+rr6+nevbvgHcHGNSkSiThw4AD79u3jgw8+YM+ePdja2iIWizl+/Dg//vijomJhI0IMRZZIJJw7d45t27ahp6dHbW0ttbW12NnZIZPJKC4uJjk5mW7dutG/f38AQV+Uvl13QFNTk8rKSoyNjdm+fTvdunXjwIEDlJeXM2PGDDw9PRWfFaI9odxqyt/fny1btiAWi5vsUw4ODvj7+ysuZdT8Y/5yzqDyi018fDz6+vr88ssvmJqaKnIBG/sAtW3bFh8fH1XKVTkxMTEsWLAAuVyOt7c3Ojo6ZGRkkJycTEhICMXFxSQlJfH69Wvs7OwYP368qiWrHLlcTnV1NWPHjsXQ0JBJkyYhk8m4fv06QUFBxMTEUFFRwbhx4wDw9fWld+/eSKVSwW3cyusxNzeX+/fv069fP4qKirh27Zqiz0/nzp0JDQ0VfL6u8ngVFBRgbGyMra0tq1evpm/fvshkMkX4FSDo3C349UXw2bNnrFu3jsePHzNr1iwCAgJ49eoVO3bsYNGiRYSEhBAZGdmklYtQUTY8X7x4wfr162nVqpUivHHq1Kk8e/aM6Oho+vbti7GxsYoVqx7lfXv79u0MGjRIUbRj6dKlRERE0KVLF3r37k2HDh0AYbaOaEQsFnPx4kUuXLhAQEAAY8eO5cyZM5SWluLg4ICvry/h4eGK/ovKL/tCQ9kJPnjwINeuXaOyspIWLVqQkJCAv78/Xl5epKam0qpVK0UVTKH2EGz8zlFRURw7dgxdXV369u3LunXraN68eZOoD01NTVXJ/NPxl3MGGxfVN998w+nTpzE1NcXJyYkNGzYomqNfvXqVwYMHY2RkpGK1qicrK4urV69y7do1ioqK0NfXJzAwkNLSUlxdXZHL5UgkEiZMmKA45IS4CZWXlyti0hsrhlZVVXH27FmFUWBtbY1MJkNHR4fPP/8c+DVvQqghCmKxmMrKSr766iuMjIx4/vw5M2fOREtLi1WrVnHo0CGePn1K27Zt0dDQENy8epvG/WvmzJmKm/UBAwZgZ2fHokWLGDRokGCNpt9DJBJx//59vv/+e1xdXXnx4gXnz5+ne/futG3blgsXLlBaWtokhEjoNBbzmDx5Mra2tnTt2pWlS5cyevRoHB0dycrKIiEhgWXLlmFtba1quSqn8bybM2cOr1+/xtTUlNu3b9OhQwfs7Ow4duwYRUVFREREKGwKITqCynaBSCTCxcWFoKAgzp49i4mJCZ07d2bHjh1YWlpiZ2fXZB8T2lgp0zgO+/fvJy4uDplMRnp6OkVFRdTW1pKQkMDWrVuZOnUq4eHhis8Jaczq6+spLi5GV1cXgMWLF1NYWEjPnj356quv6NmzJz4+PixdupS2bduqbfv/AJFcudHSX4R9+/Zx7do1li1bRk1NDVKplGXLluHq6kp6ejqffvqpuqT4f1NTU8PZs2e5cOECBgYGNG/enF9++QV7e3s++ugjTE1NqaysxMTEBBDmIVdeXs6xY8cICgrC1NRUsSHBm8by2dnZbN68+TefE2IsP0BGRgZZWVno6+vj4eHBoUOH2LNnD3FxcQBMnjyZhoYGjIyMqKioYPHixYIcp9+jvr6eGTNmYGhoyNdff83JkydZtGgRp0+f5vPPP8fd3Z3JkyerWuY7Q3FxMdu2bePmzZv89NNPlJeXs3jxYjQ0NPjmm29ULe+dRC6Xk5+fz4wZM/jhhx+QSCTs2LGDzZs3Exsb22R/U/PmQk8qlXLp0iVu3LhBcHAwp06doqSkBKlUiq6uLl9//bX6FYI30R+2traKnxvti2PHjuHk5ISnpyc9e/ZUocJ3B2VbaunSpSQlJbF9+3a0tLS4ePEid+7cITg4mMLCQjw8PBQvXkKzweRyOTExMQQEBODm5saLFy/YtGkT//Vf/8Xly5e5fPkyqampbN++nVu3bhEcHCz4QpD/CX+Zl0Hlp/a7d+9iZGSEn58fEomEPXv24OzsTM+ePQkODlZU5FPzJrbf3Nyc0tJSnjx5wpgxYygpKeHUqVNUVVUREhKCrq6uYgMS0iYEkJaWho2NDWlpaXz33Xc0NDTQqlUrRbXCjh07snfvXu7evUvHjh2bfFZoYwVvcgKXLVtGdXU1z58/p0OHDshkMjIyMigqKsLPz4/u3btjbGyMjY0NY8aMEeQ4KaN8uIvFYm7fvs3gwYMxNDTE1dWV3NxcLCws+OSTTwRXkfAfcfnyZR48eICvry/p6em8fPmSNm3a4OLiwqlTp/Dx8UFfX1/w80uZ+/fvc+7cOczNzXn58iXFxcW4ublhbW3Nzp07sbGxwc3NTdUyVYryC1dOTg47d+7EysoKIyMjkpOT6devH61bt0YsFmNtbc24ceOQSCSCzHl72zGZM2cO+fn5tGzZEnhjX1hbW6Ojo0N9fb2ikI4Qo4uUaRy3iooKNDU1sbe3Z+/evbi5uSmc6aNHjzJkyBDc3d3R19dXzC8hjVvjhbqrqyu6urqcOXMGU1NTrKysiI6OxtLSkpEjR3LgwAHkcjlDhgxRX8r8h/ypncH8/HwSEhJwcnJqsgnX1dWxatUqHB0dKSkpYevWrXh6euLq6qpCte8u2traWFpa8uDBA27fvs3EiRMJDQ2lR48eit8R0gYEb2408/PzefDgAS4uLiQnJ5Oeno5MJiMkJASJRKJwCCMjI/H19RV0dVWAK1euEBMTw6JFi4iMjFSUe9bQ0MDe3p6EhARKS0tp0aIFNjY2gm+Sm5ubi46OjqKvVqOBcPjwYbKysggMDOT+/fvExMTQsWNHLC0tVaxYtbxtQMbFxZGXl4e5uTmtWrXi0qVLVFdX07ZtWzp16oSJiYng9q1/xq1bt7hx4wZSqRSxWMy9e/c4deoUO3fuZNq0aYrKokJFOZqjoqKCnJwc8vPz2bhxI66urhw4cACZTIavry9ubm6KghXKub5ConF9lZaWoq2trRiDFi1aKJwXDQ0NXFxcaNWqFSDs/MBGRCIRV69e5dtvvyUhIQFtbW26dOnCvHnzaNasGYcPH8bOzq7JBbPQxkx5LZaXl/Pw4UMOHjyImZkZfn5+PHnyBKlUOQlE4QAAHxdJREFUSlRUFP369ePjjz9WreA/OX9qZ1AmkzFjxgxFGWxXV1fq6+uxsrLCxMSEpKQkTp8+zaBBg5o4Nmp+i56eHhYWFly4cAELCwvF7bDQbjvlcjmvX79m8ODB+Pj4EBoayoIFC+jWrRt9+/bl7t27pKWl0bp1a8WhJhaLFTefQhqrt0lOTsbd3Z2AgABqa2vR0NBALpdz48YNqqursbKy4ueff6Z9+/ZoamoK3lC/efMm8+bNQ1tbG1NTU7S0tBCJRAQGBrJr1y7u3LnDrl27GD9+vOB7VMIbA6q0tJSjR4/i5eWFl5cXmZmZZGdn4+TkhImJCQ8fPlTknwqdrKws9PT0kEgkxMTE4Ovri5OTE69eveL+/fu4ubnRqVMnDAwM6N27N23btlW1ZJWi7KSsWrWK3bt3I5VKGTlyJJaWluTn55ORkYGenh7t27dvst8Lbd9X/u6HDx/m2LFjyOVyrK2tiYqKomvXrk16ECsj1H1f+TLrzp07rFq1ipUrV1JTU8OqVasYOXIkFhYWLFu2jL/97W98+OGHgPDCQhtp/M5bt25l7dq1+Pj4YGhoSEJCAgYGBrx48YKMjAyCg4MZOHCgitX++flTOoPKi6qoqIjjx4/ToUMHPDw8FDd0rq6uBAUF0blzZ8GXxf5XMTY2pm3btk1ebIR2yDU0NKClpYWZmRnLli0jMjKSO3fu8PPPP9OhQwfMzc05f/48mzdvxtPTEwsLC8VnhTZWb5OamsqVK1cIDQ1FU1NTka+7fv16bGxs6NGjB4GBgTRr1kyQh1sjjfuXo6MjmzZt4uLFi4wfPx6RSERNTQ26urqEh4fTtm1bwsPD8fPzU7VklaFsCDU0NFBRUcHEiROxsrLC09MTNzc3tm3bRlVVFcHBwXTr1k3Qc6uR0tJSrl+/zoMHDwBYs2YNaWlphIWF4eHhwZkzZ0hLS8PPz4+2bdtiZmamYsWqo66uTpEX2NDQwOzZsxGJRERERHD58mUKCwvp2rUrvr6+6OnpcezYMSIjIwV74dDQ0IBEIqGqqoqioiIA3NzcWLZsGQ4ODmRkZGBubi74yA9l6urqFK+mN2/exNzcnOfPn5Ofn098fDxjx47l1q1bDB06VPF3kZGRgHCdZ3jTKujixYvMmjULZ2dn2rZty9WrV8nKyqJLly70799f8B0B/n/xp7JeKyoqFNUZKyoqyM/PJyQkhG3btrF8+XKys7N/s0ELPXTv30W5GpoQadyw9fT0kEqlfPrpp3z++efo6uqyY8cOXFxc+Pzzzxk3bpyixLOaN7Rp0wZzc3MuXboE/FrWuby8XLEOhZ7Y3Rj6UlVVhVwu57PPPsPa2prly5cDv46Zjo4OBgYGgq/m2GgIxcbGsn79ekQiET/88AMrVqwgNTWViooKtLW18fPzUxufvGkVUVZWhoGBAS9fvmT69OlcuXKFffv2ce/ePUWhq4aGBoUzLWTu37/P9OnTmTlzJhkZGYjFYmxsbPj0009JTU0F4JdffiEqKgqAiIgIzM3NycnJUaVslSIWi3n48CETJkxgy5YtHDlyhGbNmrFq1SoaGhooKioiKSkJEK4d0UhdXZ3iUrS+vp61a9cSExNDSUkJjx49IjY2ljlz5lBSUsKdO3eQy+XMmDEDXV1dQc6xhoaGJj+LxWIGDx6Mm5sbhoaGHDt2jE6dOuHm5oaDg4M6P/D/I3+Kl8HG0L3PP/8cc3Nz6uvrmTJlCrdv31ZUD3J1deWrr74iKysLFxcX9PX1VS37T40Qb6MaXyE2bNjA2bNnmTlzJvfv3+f8+fPMmTOHqKgoXr9+TUhICE5OTk0+owYMDQ0pLS1VvKSam5sze/ZsbGxsGD58uKrlvROIRCIyMjKYOnUq+fn5dOzYkSFDhjBnzhwASkpKcHBwEPwrszL79+/n0KFDDBkyhPT0dDp06ICNjQ3r1q1j7969DB8+nC5duqhapso5ffo0S5cu5cKFC+Tn5xMcHKx4aba2tqZXr16sX7+ew4cPKxweIRMfH8/q1avp168fGhoarF+/noEDB1JaWsrZs2fx9fVl0KBBnD59Gg8PD3x9fXn48CFpaWkMHDhQUPu+8jlXU1PDvHnzGDVqFB4eHhw5coTq6moCAwMVqRU//fQTnTt3/h9DRYXA/fv3WbRoERcuXMDBwYHr16+zY8cOZsyYQYsWLRQXWWfPniU7O5s5c+YoihtGRERgYGCg4m/wx6FcoDA7O5szZ86gr69PVVUV27Zto0ePHsjlcr7//ns8PDzo3bu32hH8/8yforVE40Q5duwYBw4cQF9fn969exMSEsLZs2eJi4tj5cqVbNu2Dblcrk4kVfNv8XYLiCVLluDt7U1ERAQAgwYNIiQkhGHDhlFdXY25ubmqpL7zNDQ08OzZMzZv3oyBgQHm5uYMHTpU1bJUirIhlZSURGxsLN7e3ty/fx9tbW0++ugjiouL+fLLL3n//fcFv38pF+Ooqalh3LhxdO3alYaGBpKSkkhOTmbLli3IZDLq6urU65E37ZQuXbrEZ599hqurq2JPq6ioYMuWLWhqatK6dWt0dHTQ19dv0phZiBw+fJjly5ezZcsWnJ2dqa+vZ+7cucyaNQuxWMzUqVMVRa969+7NsGHDVC1ZZSivx8ZcwWnTphEUFMSxY8f45JNPKCgowN/fHzs7OwoLC5k8eTJr1qzB0NBQxepVQ3x8PJs3b2bYsGEK52bnzp38/e9/x9DQkC+//BKA6upqMjIyFNVXhVqIqJGkpCSWLl2Ks7Mz586d48KFC2zcuJGCggJycnLo0KED48ePV7XMvyTv/MugcqKyu7s7AAkJCQwYMAAzMzNMTEy4ffs2wcHB+Pv7Czq/Rs2/T2PRgPz8fG7evIm2tjZPnz6ltrYWExMT9PX1ycjIID8/n169eiGTyQRfFvsfIRKJ0NfXJzQ0lPbt26vj+fn1lb26uprRo0fj6urK6NGjsbGxIT09nbS0NIKDg+nbty8BAQEqVqtaGtdjWVkZt27dwsTEBCcnJx49ekRNTQ0zZ87k3r17WFpa4u7ujp6enqolq5za2lqOHDnC0KFD8fT0JCEhgRkzZnD69GnKysqIjIwkJSWFbdu20atXL+zt7VUtWaXI5XIePXpEcXExERERNDQ0MHnyZBwdHWnfvj1isZjmzZtz79493n//fUXulhD3/cb8QICoqChu3bqFk5MT+fn5LFiwgE2bNlFRUcH69evp3LkzxsbG5OTk4OzsjKenp4rVq4bDhw8zf/58li9fTps2bWjZsiV37tyhU6dO+Pv7s3//fgoLC/Hx8UFTU1NRJVp5rIWC8ppKT09n4cKFTJ48mY8++oiysjKio6NZvHgxoaGh+Pj4qAtB/h/yTr8MKt+ob926FZlMRnh4ONHR0VRWVjJixAgSExM5dOgQq1atwsjISHCbtZr/PfHx8URFReHl5YVIJEIqlVJZWUlNTQ0PHz4kNDSUUaNGqVqmmj8x0dHRODk5YWxszJQpU9i8eTP29vYkJSVx+fJlPvroI0xMTFQt850gNzeXadOmYWBggJ6eHiNGjMDOzo5t27aRnZ0NwPfff69ake8YW7ZsISoqioiICJKSkujXrx/e3t5MmTKFmJgYzMzMqKmpoVmzZqqW+k5QU1NDbGwsR44cobS0lNGjRysMzcZiMsqvNEJOB6ivr2fp0qUUFBRgY2PDxYsXOXjwIPPmzUNXV5fU1FQWLFjQpNm8UJHL5cTGxnL06FEWLFiAtrY2X375Jd7e3kyYMAH41elZtmyZoKMa3n5xzsnJYd26dejq6jJ37lwARo0ahZ6eHqtWrVKlVEHwTjuD8GbT/vrrr7GxsUFLS4uEhARmz55NdHQ0GRkZtGnThhEjRjSp6qhGzT/i7TC0hQsXMm7cOB49esSyZcsYNmwYnp6eaGpq8vTpUzp06AD8NpxUjZr/ibeNx3PnzrFlyxYWLlxIUlIS27dvJyYmBl1dXWpqagSd/6A8Vnfu3GH//v20a9eOiIgIdu/eTWpqKh9//DGFhYU8fvxY0bhaTVMuXLiAvr4+hoaGODs7AzB9+nQmTJiAnZ2ditW9e5SXlxMdHU1ycjI//vgjgODX4tvU1tayfft2fv75Z7Zu3QrAggULyMvLY8OGDQBNKrGqz8d/7aKhvLxc0MXUGvf8mpoaRcGc0NBQTE1NOXbsGM2bN2fEiBEAFBQUCLra8R/FOxcmmpOTw7Nnz5BKpWhra5OTk8Pt27eZNWsWx44dA8DExIThw4fz+vVrJk2aJOhFpebfQzkUY+/evbx8+VLRvDohIYFp06aRmZmJra0tbm5uCiNKfdCp+XcQiUQUFRVx9uxZzMzM8PLyQiwWs3nzZiZNmsSDBw/IyMggICBAcKFByrzdmzM1NZX4+Hg8PDxwcXHBwsKC7Oxsrl27xvDhw/H29lah2ncbBwcHpFIpubm5aGpqMmXKFCwsLOjVq5eqpb2TaGpq4uTkxIsXLzh//jzBwcGCXovQNGzvxYsX6OvrU1RURFFREQUFBXh7e9OxY0f27NmDWCzGw8MDsVisPh+VkEgk2NjY8Pz5c0pLS5kyZQrwxklsrHavoaEh2JdmeHM+vnz5ks8++4yOHTuira3N7du3cXBwwMXFhRMnTmBqaoqNjY06FeAP4p1yBmNjY1m4cCG5ubkcP36cgoIC7OzsuH79OitWrFA04tyyZQtBQUEEBQUJekGp+depqalBLpcrXgWXLFnCtWvXmDRpEmKxmDVr1rB48WIePnzIiRMn6N27d5OQKvU8U/PPeDun6O7du1y8eJH6+nrs7e1p0aIF586dIy4ujiVLlgg+P1D5Ymbz5s0kJCTQsmVLamtrycvLw8TEBAcHB5o3b07r1q3VFaL/BVJSUti+fTtnz54lKChI8BVD/xl6enqYmZnxyy+/YGpqipWVlaolqZTG/evAgQOsXbuW+/fvU1FRgbu7O8+fP6eoqAhXV1f69u1LixYtfvM5NW/4ZxcNQhyvt8/HR48eIZVK+eCDD4iNjSU3N5cnT55gYmJCZGSkunXXH8w74wyuW7eOxMREvvnmGwYOHIiFhQUPHjwgLS0Na2trqqurMTMzY9GiRYSHh9OmTRtVS1bzJ+H+/ft8++23xMXF8eLFCywtLUlKSqKoqIg+ffrQokULJBIJd+7cITU1le+//x4TExNB54mo+fdoLHwCcPz4cS5evIitrS3V1dUUFBRQXV1NYWEhRUVFdO3aVd0TDxRhQtOmTaOqqgpTU1MuXbpEy5YtKS4u5vr167i7u2NjY6N2BP8FRCIRNjY2RERE0LFjRwIDA1Ut6U+BsbExbdq0wdHRUdVSVEZcXBy2trZIJBLi4uLYs2cPCxcuRE9Pj7y8PBoaGmjWrBmZmZn4+vqiqamJSCQSZFGdfxX1RcOvKL8c37lzh/r6ehwdHXn8+DE7duxg+vTpNG/enEuXLtGpUyd1BIgKeCdyBuPj4/nss8/Ytm0bXl5eAIoy4levXqVXr168evWKxMRE2rRpQ+vWrVWsWM2fhfj4eDZt2sTHH3+MiYkJVlZWmJmZkZuby86dO9HU1GTy5MnAm/yIxjAOoZd4VvOfsXTpUjIzM/H396e4uBhzc3PkcjlZWVncvn2bBQsW0KpVK1XLVBkZGRlkZWWhr69Pq1atyMrKYs2aNYr8o507d5KcnKyohvnBBx+oWLEaISHEC8CNGzeyfv16pk6dyocffkhMTAwvX75k4sSJAGzatAkzMzO6du1KbW2tYNtF/Ke8fPkSIyMjVctQGcprau3atZw+fRpbW1uCgoIYNmwYAwYMoEuXLpw+fZpp06bRvn17FSsWJu/Ey6CxsTE6OjokJibSsmVLtLW1EYlEGBgYsGLFCsLCwvDx8aFNmzY0b95c1XLV/Ik4evQo/fv3JygoCAsLC/T09GhoaCA7OxsXFxcSExPJzs7G39+/SeU4df6Dmn8F5Zvx/Px8Dh06RFRUFG3atKG6upqUlBRGjhxJWFgYkZGRODk5qVix6jh58iTLli2jurqaZ8+eKQoz3bp1Cz09PWxtbTExMeHixYtEREQoem+pUfNHITRHEMDKyorbt29TUlJCTU0N/v7+7N27FwcHBywtLdm9ezcaGhq89957aGtrC9Jh/t+go6MDCO+ioby8nEuXLmFnZ4dEIiEmJoZHjx7x448/4uTkRExMDBYWFgQGBpKYmMj06dPVoaEq5J2weGUyGe+//z66urpERUXR+Fgpk8lwc3NTLCY1av4d6uvrSUpKoqysDHhjuDc0NFBQUMDhw4eRSqW/29tNSBu2mv+ct3tU6urqIpVKOXnyJAAhISE8efKE8vJy9PT0MDY2VrFi1XHlyhX27dvHihUrmDNnDl9++aWifL+rqyuxsbGsXLmSTz/9lJCQELS0tFQtWY2avyyvX79W/LeOjg6tWrWie/fupKSkkJqayuDBg5k5cybjxo1DR0enSaNv9fn4nyGkccvMzGTJkiXIZDLkcjkPHz4kIyMDuVxOUVERLVq0YPjw4cyfPx9HR0e+++47RQVkNarhnXgZhDfx1VZWVly/fp2CggLs7e2ZPn06ZmZm9OzZU9Xy1PwJEYvFVFVVkZOTg6urK7q6uohEImQyGbt376ZXr164ubmp25Ko+Y8QiUQkJCQwefJkysrKqKysxNXVldOnT/P06VM2bNiAt7c33bp1U7VUlZOcnIy7uzsBAQHU1tYilUqRy+Xcu3cPiURCYGAgBQUFDB48mPDwcFXLVaPmL0tMTAwLFixALpfj7e2Njo4OGRkZpKSkEBYWxs8//0xISAgjRozA19eXAQMGAL+t/KtGze/R0NDA8uXLMTExoX///kyaNAkTExMCAgJ48uQJFRUVWFtb4+7ujqurK+7u7qqWrIZ3yBkEMDIywsTEhF27drFy5UoGDRrE6NGjVS1LzZ8YHR0dUlJSeP78OU5OTmhqarJkyRJev35N7969kUqlqpao5k+EclhoWVkZa9eu5YsvvmDo0KG0aNGC1NRU3n//fR4/fkybNm346KOPVKz43SA1NZUrV64QGhqKpqYmNTU1SKVS1qxZg52dHcHBwfj7+wu6yIIaNX8EWVlZXL16lWvXrlFUVIS+vj7t27enpKQENzc3KioquHDhAp06dVI0RVeu/KtGzT9CJBKhq6vLqVOnSE9PJzQ0lL1799KuXTuMjY1JSkpSVKi1trZWtVw1/807Zwl7e3szfPhwNDU11dXQ1PyvcXZ2pnv37pw4cYLx48djaGiIpaUlq1atAoQXx6/mP0e5qFBeXh6WlpZoaGjw5MkTRVGY2NhYwsPDGTlypCqlvnO0adOGBw8ecOnSJbp27apo7F1RUaHoI6Vei2rU/N/To0cPNDQ0uHDhApWVlSQmJrJy5Urs7e3p0KEDQ4cOpby8vEmotvpFUM2/Q319PVlZWZiYmNCpUyekUimbNm3i888/x8fHBy8vL/Ve/47xzjmDYrGYkJAQVctQ8xfCx8cHHx8f8vLyqKqqwsXFBVBXDFXz7yGRSGhoaGDSpElIpVJatGhBWFgYV69eRVtbm7y8PDQ0NNT5br+Dvb093t7e3L59m5s3b9KvXz9Wr16NjY0NoaGhgLByatSoURWampoEBQVRXFxMWloa/fv3p66ujv3796Ojo8PUqVPVhWLU/Fsot44ACAwMZOPGjSQkJLB27Vo+/fRTMjMz2b9/P9OmTVNUbVfz7vBOtJZQo+aP5u3NS42af0ZdXR1r1qyhoaGBKVOmUFdXx61bt6itrSU2NhYNDQ2mTp2qeOlS05SGhgaePXvG5s2bMTAwwNzcnKFDh6palho1guTFixf89NNPiMViJk+eTG5uLra2tqqWpeZPRqMtVVxcTHJyMmFhYYp/S0lJ4dy5c2hoaDBx4kSqq6vVl6XvKO/cy6AaNX8EakdQzT/j7ZtxqVRKRUWFouqZVCpl8+bNTJkyhdmzZ6tvO/8JYrEYa2trZs+erWopatQIHnNzc/r06cPGjRtJTk5WtHJRR8yo+XcQi8U8fPiQDRs2KFpDNDqIvr6+vHz5klevXgGoHcF3GLUzqEaNGjVvoewIHj9+nKdPn9K6dWuCg4O5cuUK169fx8HBgaqqKjQ0NNSOoBo1av50ODs7M2PGjCZN0dWOoJp/xtuRVbt27SI/P59evXoBbxzExjO0Y8eO6nDjPwHqMFE1atSo+R9Yt24dqampeHp68vLlS3x9fdHU1OTIkSPU1dUxYMAAevTooWqZatSoUfO/Qp0jqOZfQdkRvHLlCqamptja2vL111/TunVrhg0bpq7S/idE/X9MjRo1an6HvLw8Xr58yYYNG9i9eze3bt1CLBbTq1cv1qxZw/9r725CqlzXMI5ftvzI0Cwt0yWpqfRlaAsxSVIJrTaEQR9GREKEg4SkpJDMsg9spFSEikQZzWpQmWlWhCamkRWZoYYOShPTSNNSdJku92Cf4+nAHpxzOvqW6/+bLdbkmr1c73vfzzM4OGjXF8kDmDkogvhP/LMIHj9+XFarVaOjozKbzUpOTlZhYaHc3d21bds2g1Piv8XiFAD8DVdXVwUHB+v8+fOy2WxKTU1Va2urampq5OTkRBEEANiFH4cIa2tr5eDgoNzcXC1cuFBv3ryRq6urUlNT1dDQoLGxMQOT4n9BGQSAf/jxgefl5aVNmzapt7dX8fHxunXrltasWaO0tDTeogMAZrzx8XFJf3057u/vlyT5+Pior69PycnJCg0NVV5ensrKyhQZGakzZ84wJvobogwCsFuNjY1qaGiQJD169Ei3b9/+t/89PT0nr5Lw9fXVgQMHjIgJAMC0qq2t1bVr1yRJ1dXV2rdvnwoKCvTq1SsFBQXJxcVFwcHBysrKkqenJy9Jf2McIAPALn379k2FhYVyc3PTzp07VVlZKbPZrJiYGEnS6OionJ2dJUl9fX2MhQIA7MLQ0JC+fPmiI0eOKDo6Wu3t7dqyZYvev3+vgYEBmc1mzZs3T1VVVYqOjtbmzZuNjoyfYDp16tQpo0MAwHQaHx/X7NmztWDBArW0tKizs1OPHz/W2NjY5EPOZrNNHrPu6upqcGIAAKbewMCAHjx4oMDAQC1atEhlZWWyWCxKSkqSh4eHPn78qI6ODm3dulUJCQlasWKF0ZHxkxgTBWBXJiYmZDKZZLVaFRISoj/++EM9PT3q6elRU1OTioqKlJycrNLSUqOjAgAwrdzd3dXd3a09e/Zo1qxZSklJUX19vdrb27VkyRJFRERo8eLFMplMk9Mz+L2x5QnArjg4OKitrU05OTmy2WzKyspSbGysBgYGFBsbqw0bNqihoUERERFGRwUAYMo1Nzfr5cuXcnJyUlRUlHx8fOTl5aV3795p//796uzs1Llz55SVlaWwsDCFhYUZHRn/R3wZBGBXXrx4ofz8fB06dEgJCQnKzc2Vv7+/wsPDVV9fr66uLoogAMAuVFRUKCcnRyaTST09PTpx4oT8/PyUmZmp4eFh3bx5Uzt27JDValVzc7PRcTEFOEAGwIw2Pj4+ufs3MTGhCxcuqKKiQg8fPpQk5eXl6cOHDzp27JgGBwcVHBxsZFwAAKbF8+fPVVRUpJMnT8rf31+SVF5ersrKSqWnp6uzs1MlJSXq6upSdna2QkJCDE6MqcABMgBmrB+L4J07d+To6KioqCg1NTXp9evXWrdunaKjo/XkyRNFREQoICDA4MQAAEyP5uZmeXt7KyYmRlarVY6Ojlq6dKnq6urU0tKiXbt2aeXKlYqIiNDy5cuNjospwpgogBnLZDJpeHhYGRkZKi0t1fXr11VdXa2jR4+qo6ND+fn5kqSzZ8/Kz8/P4LQAAEyfkZERPXv2TKOjo3JxcdHw8LAkKSYmRl5eXpIks9ms0NBQI2NiilEGAcwoPT09slqtk7+Li4tlsVhUWFio/v5+1dXVqbW1VWlpafr+/bukv8ZHAQCwJ5GRkQoICFBlZaWkf12jdP/+fc2ZM8fIaJhG7AwCmDHu3bunoqIirV69Wq6ursrMzNTdu3c1d+5clZaWKj4+XjU1Nfr06ZMyMzPZfwAA2LXy8nI1NjZqZGREiYmJunz5soKCgpSRkWF0NEwTyiCAGaGgoEBtbW1KSUnR/PnzdeXKFWVlZWl0dFS1tbUaGhrS+vXrlZ6err179yomJsboyAAAGMpms6m7u1tXr16Vr6+v3N3dlZSUZHQsTCPKIIDfXm1trQ4ePKji4mKFhYXp6dOnSk9P18aNGxUYGCiLxaK0tDR5enrq8OHDiouLMzoyAACA4SiDAH57g4ODunHjhnp7e7Vq1SqVlJQoLi5O4eHh2r17t6qqqvT161d5eHjI09PT6LgAAAC/BEejAwDAz3Jzc1NiYqIuXbqk06dPq6ioSBaLRZIUGxurz58/a9myZQanBAAA+LVQBgHMCN7e3kpOTtbIyIi6uroUFBSk7Oxsmc1miiAAAMDfYEwUwIzS2Nioixcv6u3bt0pPT9f27duNjgQAAPBLogwCmFFsNptqamrk7OystWvXGh0HAADgl0UZBAAAAAA7NMvoAAAAAACA6UcZBAAAAAA7RBkEAAAAADtEGQQAAAAAO0QZBAAAAAA7RBkEAAAAADtEGQQAAAAAO0QZBAAAAAA79CcbS3xVYcFGQAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x576 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from UtilTextClassification import plot_freq\n",
    "\n",
    "# Label distributed fequency.\n",
    "plot_freq(df, col=['product'], top_classes=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Demo of product and its complaint example...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>consumer_complaint_narrative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Debt collection</td>\n",
       "      <td>XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Consumer Loan</td>\n",
       "      <td>Due to inconsistencies in the amount owed that I was told by M &amp; T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M &amp; T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX X...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mortgage</td>\n",
       "      <td>In XX/XX/XXXX my wages that I earned at my job decreased by almost half, by XX/XX/XXXX I knew I was in trouble with my home loan. I began contacting WFB whom my home loan is with, for assitance and options. \\nIn early XX/XX/XXXX I began the Loan Modification process with Wells Fargo Bank. I was told that they would not assist me with anything financial on my home loan until I fell 90 days behind, though at the time I started to inquire for assistance from WFB I was only a few weeks behind. So, I began working with a program called XXXX. They approved me for a variety of assistence and reached out to Wells Fargo Bank to determine what they could assist with. Wells Fargo then turned down the assistance from XXXX and finally offered to do a Loan Modification for me. The outcome was totally unknow about what I would be offered in the end by WFB for assistance. Wells Fargo lost my paperwork twice during this process, so it took 2 months from the time I started to the time my paperwork b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Mortgage</td>\n",
       "      <td>I have an open and current mortgage with Chase Bank # XXXX. Chase is reporting the loan payments to XXXX but XXXX is surpressing the information and reporting the loan as Discharged in BK. This mortgage was reaffirmed in a Chapter XXXX BK discharged dated XXXX/XXXX/2013. Chase keeps referring to BK Law for Chapter XXXX and we keep providing documentation for Chapter XXXX, and the account should be open and current with all the payments \\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mortgage</td>\n",
       "      <td>XXXX was submitted XX/XX/XXXX. At the time I submitted this complaint, I had dealt with Rushmore Mortgage directly endeavoring to get them to stop the continuous daily calls I was receiving trying to collect on a mortgage for which I was not responsible due to bankruptcy. They denied having knowledge of the bankruptcy, even though I had spoken with them about it repeatedly and had written them repeatedly referencing the bankruptcy requesting them to cease the pursuit, they continued to do so. When they were unable to trick me into paying, force me into paying in retaliation they placed reported to my credit bureaus a past due mortgage amount that had been discharged in Federal Court. On XX/XX/XXXX Rushmore responded the referenced complaint indicating that they would remove the reporting from my bureau, yet it is still there now in XX/XX/XXXX. I would like them to remove it immediately and send me a letter indicating that it should not have been there in the first place and they ar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           product  \\\n",
       "0  Debt collection   \n",
       "1    Consumer Loan   \n",
       "2         Mortgage   \n",
       "3         Mortgage   \n",
       "4         Mortgage   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              consumer_complaint_narrative  \n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n  \n",
       "1  Due to inconsistencies in the amount owed that I was told by M & T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M & T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX X...  \n",
       "2  In XX/XX/XXXX my wages that I earned at my job decreased by almost half, by XX/XX/XXXX I knew I was in trouble with my home loan. I began contacting WFB whom my home loan is with, for assitance and options. \\nIn early XX/XX/XXXX I began the Loan Modification process with Wells Fargo Bank. I was told that they would not assist me with anything financial on my home loan until I fell 90 days behind, though at the time I started to inquire for assistance from WFB I was only a few weeks behind. So, I began working with a program called XXXX. They approved me for a variety of assistence and reached out to Wells Fargo Bank to determine what they could assist with. Wells Fargo then turned down the assistance from XXXX and finally offered to do a Loan Modification for me. The outcome was totally unknow about what I would be offered in the end by WFB for assistance. Wells Fargo lost my paperwork twice during this process, so it took 2 months from the time I started to the time my paperwork b...  \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               I have an open and current mortgage with Chase Bank # XXXX. Chase is reporting the loan payments to XXXX but XXXX is surpressing the information and reporting the loan as Discharged in BK. This mortgage was reaffirmed in a Chapter XXXX BK discharged dated XXXX/XXXX/2013. Chase keeps referring to BK Law for Chapter XXXX and we keep providing documentation for Chapter XXXX, and the account should be open and current with all the payments \\n  \n",
       "4  XXXX was submitted XX/XX/XXXX. At the time I submitted this complaint, I had dealt with Rushmore Mortgage directly endeavoring to get them to stop the continuous daily calls I was receiving trying to collect on a mortgage for which I was not responsible due to bankruptcy. They denied having knowledge of the bankruptcy, even though I had spoken with them about it repeatedly and had written them repeatedly referencing the bankruptcy requesting them to cease the pursuit, they continued to do so. When they were unable to trick me into paying, force me into paying in retaliation they placed reported to my credit bureaus a past due mortgage amount that had been discharged in Federal Court. On XX/XX/XXXX Rushmore responded the referenced complaint indicating that they would remove the reporting from my bureau, yet it is still there now in XX/XX/XXXX. I would like them to remove it immediately and send me a letter indicating that it should not have been there in the first place and they ar...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Demo of product and its complaint example...')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Train/Valid/Testing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique values of labels...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Debt collection',\n",
       " 'Consumer Loan',\n",
       " 'Mortgage',\n",
       " 'Credit card',\n",
       " 'Credit reporting',\n",
       " 'Student loan',\n",
       " 'Bank account or service',\n",
       " 'Payday loan',\n",
       " 'Money transfers',\n",
       " 'Other financial service',\n",
       " 'Prepaid card']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Unique values of labels...')\n",
    "display(list(df['product'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from UtilTextClassification import split_size\n",
    "\n",
    "\n",
    "# Select label and label value.\n",
    "col = 'product'\n",
    "col_vals = list(df[col].unique())\n",
    "text_col = 'consumer_complaint_narrative'\n",
    "\n",
    "# Select number of data.\n",
    "train_num, valid_num, test_num = split_size(df, train=0.6, valid=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of train_X: (3000,)\n",
      "Shape of valid_X: (1000,)\n",
      "Shape of text_X: (1000,)\n"
     ]
    }
   ],
   "source": [
    "# Prepare test dataset.\n",
    "train_X, test_X, train_y, test_y = train_test_split(df[text_col],\n",
    "                                                df[col],\n",
    "                                                test_size=test_num,\n",
    "                                                random_state=1,\n",
    "                                                stratify=df[col])\n",
    "\n",
    "# Prepare valid dataset.\n",
    "if valid_num != 0:\n",
    "    train_X, valid_X, train_y, valid_y = train_test_split(train_X,\n",
    "                                                  train_y,\n",
    "                                                  test_size=valid_num,\n",
    "                                                  random_state=1,\n",
    "                                                  stratify=train_y)\n",
    "\n",
    "\n",
    "print('Shape of train_X: {}'.format(train_X.shape))\n",
    "print('Shape of valid_X: {}'.format(valid_X.shape if 'valid_X' in vars() else (0,0)))\n",
    "print('Shape of text_X: {}'.format(test_X.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Dataset to Spacy Compatible Format\n",
    "\n",
    "<mark>**Label and Text**</mark>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hard-encode label.\n",
    "train_y_df = pd.get_dummies(train_y)\n",
    "valid_y_df = pd.get_dummies(valid_y)\n",
    "test_y_df = pd.get_dummies(test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "One-hard-encoded label...\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bank account or service</th>\n",
       "      <th>Consumer Loan</th>\n",
       "      <th>Credit card</th>\n",
       "      <th>Credit reporting</th>\n",
       "      <th>Debt collection</th>\n",
       "      <th>Money transfers</th>\n",
       "      <th>Mortgage</th>\n",
       "      <th>Other financial service</th>\n",
       "      <th>Payday loan</th>\n",
       "      <th>Prepaid card</th>\n",
       "      <th>Student loan</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1291</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4421</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3875</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2935</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1173</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Bank account or service  Consumer Loan  Credit card  Credit reporting  \\\n",
       "1291                        0              0            0                 0   \n",
       "4421                        0              0            0                 0   \n",
       "3875                        0              0            0                 0   \n",
       "2935                        0              0            1                 0   \n",
       "1173                        0              0            0                 1   \n",
       "\n",
       "      Debt collection  Money transfers  Mortgage  Other financial service  \\\n",
       "1291                0                0         1                        0   \n",
       "4421                0                0         1                        0   \n",
       "3875                0                0         1                        0   \n",
       "2935                0                0         0                        0   \n",
       "1173                0                0         0                        0   \n",
       "\n",
       "      Payday loan  Prepaid card  Student loan  \n",
       "1291            0             0             0  \n",
       "4421            0             0             0  \n",
       "3875            0             0             0  \n",
       "2935            0             0             0  \n",
       "1173            0             0             0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('One-hard-encoded label...')\n",
    "display(train_y_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from UtilTextClassification import df2list\n",
    "\n",
    "\n",
    "\n",
    "# Convert to list.\n",
    "train_ls = df2list(train_X, train_y_df)\n",
    "valid_ls = df2list(valid_X, valid_y_df)\n",
    "test_ls = df2list(test_X, test_y_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preview of training list for Spacy...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('I renegotiated a mortgage wth M & T Bank during a bankruptcy proceeding.I have made prompt payments for over two and a half years on the new loan. The Bank refuses to send any information to the credit reporting bureaus depriving me of any credit rating. The bank during this time tried to collect illegal fees of over {$6000.00} that I had to hire an Attorney to get them to rescind. They now want a recommitment agreement which would put that fee back in play in order for them to report to the credit bureaus. \\n',\n",
       "  {'cats': {'Bank account or service': 0,\n",
       "    'Consumer Loan': 0,\n",
       "    'Credit card': 0,\n",
       "    'Credit reporting': 0,\n",
       "    'Debt collection': 0,\n",
       "    'Money transfers': 0,\n",
       "    'Mortgage': 1,\n",
       "    'Other financial service': 0,\n",
       "    'Payday loan': 0,\n",
       "    'Prepaid card': 0,\n",
       "    'Student loan': 0}}),\n",
       " ('I have been trying to refinance with Quicken Loans since XXXX/XXXX/2015 the process has been a painful one. they have been disrespectful, changing my numbers ( the good Faith estimate ) XXXX times even on the day of the closing. In addition to all of that, I did the so called closing on XXXX/XXXX/2015 and I signed all the papers including the HUD. They called me today XXXX/XXXX/2015 saying my closing is cancelled and I have to re -close. \\n',\n",
       "  {'cats': {'Bank account or service': 0,\n",
       "    'Consumer Loan': 0,\n",
       "    'Credit card': 0,\n",
       "    'Credit reporting': 0,\n",
       "    'Debt collection': 0,\n",
       "    'Money transfers': 0,\n",
       "    'Mortgage': 1,\n",
       "    'Other financial service': 0,\n",
       "    'Payday loan': 0,\n",
       "    'Prepaid card': 0,\n",
       "    'Student loan': 0}})]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Preview of training list for Spacy...')\n",
    "display(train_ls[:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate Text and Label on Valid/Test Dataset\n",
    "\n",
    "For the ease of evaluating model's performance later on, I split labels and texts apart on both valid and text set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert valid text and label to list.\n",
    "valid_text, valid_label = list(zip(*valid_ls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert test text and label to list.\n",
    "test_text, test_label = list(zip(*test_ls))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Demo of how valid_text is like?\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('I had a Tax Lien which was incorrectly placed when the XXXX sent tax payment money to the IRS as I had paid two years taxes ( a XXXX and a XXXX ) with the same check. They processed the XXXX return and then found they were short the money they sent to the IRS - go figure. \\nI have had the tax lien released XX/XX/XXXX and notified Experian in both XX/XX/XXXX and XX/XX/XXXX. It is still on my credit report. Again, the credit dispute process is a joke. There are no penalties for the credit bureau putting erroneous information nor in simply ignoring disputes made through their website or by mail. \\n',)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Demo of how valid_text is like?')\n",
    "display(valid_text[:1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Demo of how valid_label is like?\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'cats': {'Bank account or service': 0,\n",
       "   'Consumer Loan': 0,\n",
       "   'Credit card': 0,\n",
       "   'Credit reporting': 1,\n",
       "   'Debt collection': 0,\n",
       "   'Money transfers': 0,\n",
       "   'Mortgage': 0,\n",
       "   'Other financial service': 0,\n",
       "   'Payday loan': 0,\n",
       "   'Prepaid card': 0,\n",
       "   'Student loan': 0}},)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Demo of how valid_label is like?')\n",
    "display(valid_label[:1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct Spacy Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'textcat' not in nlp.pipe_names:\n",
    "    textcat = nlp.create_pipe('textcat')\n",
    "    nlp.add_pipe(textcat, last=True)\n",
    "# otherwise, get it, so we can add labels to it\n",
    "else:\n",
    "    textcat = nlp.get_pipe('textcat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Debt collection', 'Consumer Loan', 'Mortgage', 'Credit card', 'Credit reporting', 'Student loan', 'Bank account or service', 'Payday loan', 'Money transfers', 'Other financial service', 'Prepaid card')\n"
     ]
    }
   ],
   "source": [
    "# add label to text classifier\n",
    "for _, col_val in enumerate(col_vals):\n",
    "    textcat.add_label(col_val)\n",
    "\n",
    "print(textcat.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.util import minibatch, compounding\n",
    "from sklearn.metrics import f1_score, accuracy_score, classification_report\n",
    "import random\n",
    "\n",
    "import warnings \n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path of saved model\n",
    "\n",
    "output_dir = os.path.join(dir_path,'spacy-saved-model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the model...\n",
      "textcat loss: 22.1848\tf1 score: 0.559\taccuracy: 0.631\n",
      "textcat loss: 12.5292\tf1 score: 0.661\taccuracy: 0.712\n",
      "textcat loss: 9.4454\tf1 score: 0.685\taccuracy: 0.726\n",
      "textcat loss: 7.5352\tf1 score: 0.738\taccuracy: 0.766\n",
      "textcat loss: 6.4887\tf1 score: 0.751\taccuracy: 0.775\n",
      "textcat loss: 4.9598\tf1 score: 0.770\taccuracy: 0.789\n",
      "textcat loss: 4.3186\tf1 score: 0.774\taccuracy: 0.789\n",
      "textcat loss: 3.7593\tf1 score: 0.784\taccuracy: 0.797\n",
      "textcat loss: 3.2203\tf1 score: 0.785\taccuracy: 0.798\n",
      "textcat loss: 2.9446\tf1 score: 0.777\taccuracy: 0.790\n",
      "textcat loss: 2.5670\tf1 score: 0.776\taccuracy: 0.790\n",
      "textcat loss: 2.2185\tf1 score: 0.777\taccuracy: 0.789\n",
      "textcat loss: 1.8137\tf1 score: 0.782\taccuracy: 0.793\n",
      "textcat loss: 1.7611\tf1 score: 0.783\taccuracy: 0.794\n",
      "Finished training...\n"
     ]
    }
   ],
   "source": [
    "n_iter = 20\n",
    "print_every= 1\n",
    "not_improve = 5 \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Train model\n",
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']\n",
    "with nlp.disable_pipes(*other_pipes):  # only train textcat\n",
    "    optimizer = nlp.begin_training() # initiate a new model with random weights\n",
    "    print(\"Training the model...\")\n",
    "    \n",
    "    score_f1_best = 0\n",
    "    early_stop = 0\n",
    "    \n",
    "    for i in range(n_iter):\n",
    "        losses = {}\n",
    "        true_labels = list() # true label\n",
    "        pdt_labels = list() # predict label\n",
    "        \n",
    "        # batch up the examples using spaCy's minibatch\n",
    "        random.shuffle(train_ls)  # shuffle training data every iteration\n",
    "        batches = minibatch(train_ls, size=compounding(4., 32., 1.001))\n",
    "        for batch in batches:\n",
    "            texts, annotations = zip(*batch)\n",
    "            nlp.update(texts, annotations, sgd=optimizer, drop=0.2,\n",
    "                       losses=losses)\n",
    "            \n",
    "        with textcat.model.use_params(optimizer.averages): \n",
    "            # evaluate on valid_text, valid_label\n",
    "            docs = [nlp.tokenizer(text) for text in valid_text]\n",
    "            \n",
    "            for j, doc in enumerate(textcat.pipe(docs)):\n",
    "                true_series = pd.Series(valid_label[j]['cats'])\n",
    "                true_label = true_series.idxmax()  # idxmax() is the new version of argmax() \n",
    "                true_labels.append(true_label)\n",
    "    \n",
    "                pdt_series = pd.Series(doc.cats)\n",
    "                pdt_label = pdt_series.idxmax()  # idxmax() is the new version of argmax() \n",
    "                pdt_labels.append(pdt_label)\n",
    "                \n",
    "            score_f1 = f1_score(true_labels, pdt_labels, average='weighted')\n",
    "            score_ac = accuracy_score(true_labels, pdt_labels)\n",
    "            \n",
    "            if i % print_every == 0:\n",
    "                print('textcat loss: {:.4f}\\tf1 score: {:.3f}\\taccuracy: {:.3f}'.format(\n",
    "                    losses['textcat'],score_f1, score_ac))\n",
    "            \n",
    "            if score_f1 > score_f1_best:\n",
    "                early_stop = 0\n",
    "                score_f1_best = score_f1\n",
    "                with nlp.use_params(optimizer.averages):\n",
    "                    nlp.to_disk(output_dir) # save the model\n",
    "            else:\n",
    "                early_stop += 1\n",
    "            \n",
    "            if early_stop >= not_improve:\n",
    "                print('Finished training...')\n",
    "                break\n",
    "            \n",
    "            if i == n_iter:\n",
    "                print('Finished training...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load saved model.\n",
    "print(\"Loading from\", output_dir)\n",
    "nlp = spacy.load(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from UtilTextClassification import evaluate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On Validation Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 score: 0.785\taccuracy: 0.798\n",
      "\n",
      "classification report...\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "Bank account or service       0.60      0.65      0.63        81\n",
      "          Consumer Loan       0.60      0.58      0.59        53\n",
      "            Credit card       0.71      0.80      0.75       109\n",
      "       Credit reporting       0.85      0.80      0.82       162\n",
      "        Debt collection       0.81      0.85      0.83       285\n",
      "        Money transfers       0.00      0.00      0.00        11\n",
      "               Mortgage       0.89      0.96      0.92       234\n",
      "Other financial service       0.00      0.00      0.00         2\n",
      "            Payday loan       0.00      0.00      0.00        13\n",
      "           Prepaid card       0.00      0.00      0.00         6\n",
      "           Student loan       0.89      0.75      0.81        44\n",
      "\n",
      "              micro avg       0.80      0.80      0.80      1000\n",
      "              macro avg       0.49      0.49      0.49      1000\n",
      "           weighted avg       0.77      0.80      0.78      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test dataset evaluation.\n",
    "evaluate(nlp, valid_text, valid_label, label_names=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 score: 0.810\taccuracy: 0.826\n",
      "\n",
      "classification report...\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "Bank account or service       0.76      0.80      0.78        81\n",
      "          Consumer Loan       0.67      0.48      0.56        54\n",
      "            Credit card       0.75      0.86      0.80       109\n",
      "       Credit reporting       0.81      0.78      0.79       162\n",
      "        Debt collection       0.82      0.89      0.86       284\n",
      "        Money transfers       0.00      0.00      0.00        11\n",
      "               Mortgage       0.93      0.96      0.94       234\n",
      "Other financial service       0.00      0.00      0.00         2\n",
      "            Payday loan       0.00      0.00      0.00        13\n",
      "           Prepaid card       0.00      0.00      0.00         6\n",
      "           Student loan       0.82      0.84      0.83        44\n",
      "\n",
      "              micro avg       0.83      0.83      0.83      1000\n",
      "              macro avg       0.51      0.51      0.51      1000\n",
      "           weighted avg       0.80      0.83      0.81      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate(nlp, test_text, test_label, label_names=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo of Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experian is reporting a negative account that does not belong to me. This account was previously deleted from my file by the creditor as it did not belong to me. I am attaching the response from the creditor deleting this information. The original creditor is not listed but it is for the same amount as the previous account. Experian is not allowing to make any online disputes and I am not able to speak to a representative as you need a report number to do so. They continue to report erroneous information and refuse to do anything about it. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "doc = nlp(test_text[0])\n",
    "print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label probabilities... {'Debt collection': 0.00035867010592482984, 'Consumer Loan': 0.0003834842354990542, 'Mortgage': 0.004348973743617535, 'Credit card': 4.539787187241018e-05, 'Credit reporting': 0.9998676776885986, 'Student loan': 4.539787187241018e-05, 'Bank account or service': 4.539787187241018e-05, 'Payday loan': 8.534910739399493e-05, 'Money transfers': 4.539787187241018e-05, 'Other financial service': 4.539787187241018e-05, 'Prepaid card': 4.539787187241018e-05}\n"
     ]
    }
   ],
   "source": [
    "print('label probabilities...', doc.cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "true label... {'cats': {'Bank account or service': 0, 'Consumer Loan': 0, 'Credit card': 0, 'Credit reporting': 1, 'Debt collection': 0, 'Money transfers': 0, 'Mortgage': 0, 'Other financial service': 0, 'Payday loan': 0, 'Prepaid card': 0, 'Student loan': 0}}\n"
     ]
    }
   ],
   "source": [
    "print('true label...', test_label[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spacy Model Using Pre-train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Construct Pre-train Spacy Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "if 'textcat' not in nlp.pipe_names:\n",
    "    textcat = nlp.create_pipe('textcat',\n",
    "                              config={'architecture':'simple_cnn',\n",
    "                              'exclusive_classes':True})\n",
    "    nlp.add_pipe(textcat, last=True)\n",
    "# otherwise, get it, so we can add labels to it\n",
    "else:\n",
    "    textcat = nlp.get_pipe('textcat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Debt collection', 'Consumer Loan', 'Mortgage', 'Credit card', 'Credit reporting', 'Student loan', 'Bank account or service', 'Payday loan', 'Money transfers', 'Other financial service', 'Prepaid card')\n"
     ]
    }
   ],
   "source": [
    "# add label to text classifier\n",
    "for _, col_val in enumerate(col_vals):\n",
    "    textcat.add_label(col_val)\n",
    "\n",
    "print(textcat.labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = os.path.join(dir_path, 'spacy-saved-model-with-pretrain')\n",
    "pretrain_model_path = os.path.join(dir_path, 'model500.bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training the model...\n",
      "textcat loss: 17.311\tf1 score: 0.615\taccuracy: 0.661\n",
      "textcat loss: 10.838\tf1 score: 0.705\taccuracy: 0.731\n",
      "textcat loss: 7.992\tf1 score: 0.736\taccuracy: 0.757\n",
      "textcat loss: 5.978\tf1 score: 0.745\taccuracy: 0.760\n",
      "textcat loss: 4.751\tf1 score: 0.755\taccuracy: 0.768\n",
      "textcat loss: 3.548\tf1 score: 0.750\taccuracy: 0.761\n",
      "textcat loss: 3.091\tf1 score: 0.751\taccuracy: 0.761\n",
      "textcat loss: 2.401\tf1 score: 0.746\taccuracy: 0.753\n",
      "textcat loss: 1.893\tf1 score: 0.751\taccuracy: 0.759\n",
      "textcat loss: 1.598\tf1 score: 0.759\taccuracy: 0.767\n",
      "textcat loss: 1.574\tf1 score: 0.765\taccuracy: 0.773\n",
      "textcat loss: 1.311\tf1 score: 0.762\taccuracy: 0.768\n",
      "textcat loss: 1.239\tf1 score: 0.763\taccuracy: 0.770\n",
      "textcat loss: 0.828\tf1 score: 0.764\taccuracy: 0.771\n",
      "textcat loss: 0.861\tf1 score: 0.763\taccuracy: 0.770\n",
      "textcat loss: 1.042\tf1 score: 0.764\taccuracy: 0.770\n",
      "Finished training...\n"
     ]
    }
   ],
   "source": [
    "n_iter = 20\n",
    "print_every= 1\n",
    "not_improve = 5\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# Train model\n",
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']\n",
    "with nlp.disable_pipes(*other_pipes):  # only train textcat\n",
    "    optimizer = nlp.begin_training() # initiate a new model\n",
    "    \n",
    "    with open(pretrain_model_path, 'rb') as file_:\n",
    "        textcat.model.tok2vec.from_bytes(file_.read()) # load in pre-train word embedding\n",
    "    \n",
    "    print(\"Training the model...\")\n",
    "    \n",
    "    score_f1_best = 0\n",
    "    early_stop = 0\n",
    "    \n",
    "    for i in range(n_iter):\n",
    "        losses = {}\n",
    "        true_labels = list() # true label\n",
    "        pdt_labels = list() # predict label\n",
    "        \n",
    "        random.shuffle(train_ls)  # shuffle training data every iteration\n",
    "        # batch up the examples using spaCy's minibatch\n",
    "        batches = minibatch(train_ls, size=compounding(4., 32., 1.001))\n",
    "        for batch in batches:\n",
    "            texts, annotations = zip(*batch)\n",
    "            nlp.update(texts, annotations, sgd=optimizer, drop=0.2,\n",
    "                       losses=losses)\n",
    "            \n",
    "        with textcat.model.use_params(optimizer.averages):\n",
    "            # evaluate on valid_text and valid_label\n",
    "            docs = [nlp.tokenizer(text) for text in valid_text]\n",
    "            for j, doc in enumerate(textcat.pipe(docs)):\n",
    "                true_series = pd.Series(valid_label[j]['cats'])\n",
    "                true_label = true_series.idxmax()  # idxmax() is new version of argmax()\n",
    "                true_labels.append(true_label)\n",
    "    \n",
    "                pdt_series = pd.Series(doc.cats)\n",
    "                pdt_label = pdt_series.idxmax()  # idxmax() is new version of argmax()\n",
    "                pdt_labels.append(pdt_label)\n",
    "                \n",
    "            score_f1 = f1_score(true_labels, pdt_labels, average='weighted')\n",
    "            score_ac = accuracy_score(true_labels, pdt_labels)\n",
    "            \n",
    "            if i % print_every == 0:\n",
    "                print('textcat loss: {:.3f}\\tf1 score: {:.3f}\\taccuracy: {:.3f}'.format(\n",
    "                    losses['textcat'],score_f1, score_ac))\n",
    "            \n",
    "            if score_f1 > score_f1_best:\n",
    "                early_stop = 0\n",
    "                score_f1_best = score_f1\n",
    "                with nlp.use_params(optimizer.averages):\n",
    "                    nlp.to_disk(output_dir) # save the model\n",
    "            else:\n",
    "                early_stop += 1\n",
    "            \n",
    "            if early_stop >= not_improve:\n",
    "                print('Finished training...')\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load saved model.\n",
    "print(\"Loading from\", output_dir)\n",
    "nlp = spacy.load(output_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On Validation Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 score: 0.765\taccuracy: 0.773\n",
      "\n",
      "classification report...\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "Bank account or service       0.64      0.64      0.64        81\n",
      "          Consumer Loan       0.59      0.43      0.50        53\n",
      "            Credit card       0.64      0.68      0.66       109\n",
      "       Credit reporting       0.79      0.80      0.79       162\n",
      "        Debt collection       0.78      0.85      0.81       285\n",
      "        Money transfers       0.67      0.36      0.47        11\n",
      "               Mortgage       0.90      0.93      0.92       234\n",
      "Other financial service       0.00      0.00      0.00         2\n",
      "            Payday loan       0.25      0.08      0.12        13\n",
      "           Prepaid card       1.00      0.33      0.50         6\n",
      "           Student loan       0.76      0.66      0.71        44\n",
      "\n",
      "              micro avg       0.77      0.77      0.77      1000\n",
      "              macro avg       0.64      0.52      0.56      1000\n",
      "           weighted avg       0.76      0.77      0.77      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Test dataset evaluation.\n",
    "evaluate(nlp, valid_text, valid_label, label_names=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### On Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "f1 score: 0.770\taccuracy: 0.779\n",
      "\n",
      "classification report...\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "Bank account or service       0.72      0.64      0.68        81\n",
      "          Consumer Loan       0.59      0.37      0.45        54\n",
      "            Credit card       0.62      0.74      0.67       109\n",
      "       Credit reporting       0.77      0.78      0.77       162\n",
      "        Debt collection       0.81      0.87      0.84       284\n",
      "        Money transfers       0.60      0.27      0.37        11\n",
      "               Mortgage       0.90      0.91      0.90       234\n",
      "Other financial service       0.00      0.00      0.00         2\n",
      "            Payday loan       0.50      0.15      0.24        13\n",
      "           Prepaid card       0.33      0.17      0.22         6\n",
      "           Student loan       0.77      0.77      0.77        44\n",
      "\n",
      "              micro avg       0.78      0.78      0.78      1000\n",
      "              macro avg       0.60      0.52      0.54      1000\n",
      "           weighted avg       0.77      0.78      0.77      1000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "evaluate(nlp, test_text, test_label, label_names=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# (Appendix) Create Pre-train Dataset for Spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import srsly\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product</th>\n",
       "      <th>consumer_complaint_narrative</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Debt collection</td>\n",
       "      <td>XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Consumer Loan</td>\n",
       "      <td>Due to inconsistencies in the amount owed that I was told by M &amp; T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M &amp; T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX X...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           product  \\\n",
       "0  Debt collection   \n",
       "1    Consumer Loan   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              consumer_complaint_narrative  \n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n  \n",
       "1  Due to inconsistencies in the amount owed that I was told by M & T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M & T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX X...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Read in dataset.\n",
    "dir_path = '../Playground-dataset/05-Try-out-Spacy-Pretrain-dataset'\n",
    "dataset = pd.read_csv(os.path.join(dir_path,'consumer_complaints_simple.csv'), header=0, index_col=None)\n",
    "dataset = dataset[['product', 'consumer_complaint_narrative']]\n",
    "display(dataset.head(n=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experiment on smaller subset.\n",
    "dataset = dataset[:5000]\n",
    "dataset = dataset.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n',\n",
       " 'Due to inconsistencies in the amount owed that I was told by M & T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M & T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX XXXX 2015, there was a delinquent balance of {$1400.00}. The monies from the GAP insurance and my insurance company has been paid, M & T Bank says that I still owe {$620.00}, of which {$210.00} has already been paid. I contacted M & T Bank via return receipt mail, but I have not been given the courtesy of a response yet. \\n']"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "texts = list(dataset['consumer_complaint_narrative'])  # only use training dataset as pre-train dataset\n",
    "display(texts[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Spacy compatible format\n",
    "final_ls = []\n",
    "for doc in texts:\n",
    "    final_ls.append({'text':doc})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'text': 'XXXX has claimed I owe them {$27.00} for XXXX years despite the PROOF of PAYMENT I sent them : canceled check and their ownPAID INVOICE for {$27.00}! \\nThey continue to insist I owe them and collection agencies are after me. \\nHow can I stop this harassment for a bill I already paid four years ago? \\n'},\n",
       " {'text': 'Due to inconsistencies in the amount owed that I was told by M & T Bank and the amount that was reported to the credit reporting agencies, I was advised to write a good will letter in order to address the issue and request the negative entry be removed from my credit report all together. I had a vehicle that was stolen and it was declared a total loss by insurance company. The insurance company and the GAP insurancw companypaid the outstanding balance of the loan, but I was told by M & T Bank that there was still a balance due on the loan. In good faith, without having received any proof as to why there was still a balance, I made a partial payment towards the remaining debt. I then sent the goodwill letter still offering to pay the remainder of the debt, but in exchange for the removal of the negative entry on my credit report. At one point, in XXXX 2015, per my credit monitoring agency, it showed a delinquent balance of {$0.00}, but when I checked my credit report again on XXXX XXXX 2015, there was a delinquent balance of {$1400.00}. The monies from the GAP insurance and my insurance company has been paid, M & T Bank says that I still owe {$620.00}, of which {$210.00} has already been paid. I contacted M & T Bank via return receipt mail, but I have not been given the courtesy of a response yet. \\n'}]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_ls[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = [{\"text\": \"Some text\"}, {\"text\": \"More...\"}]\n",
    "srsly.write_jsonl(os.path.join(dir_path,\"text.jsonl\"), final_ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "272.865px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}