COM4509-6509_Assignment_1_aca17nb.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "Python 3.8.6 64-bit",
   "display_name": "Python 3.8.6 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "d3f879b00922bef84d6a504c20b9e6598c4699780a7a052da273c06ea702fe18"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "Assignment 1 - Nathan Brown\n",
    "\n",
    "Question 1 - Data Preprocessing - Part A\n",
    "\n",
    "The dataset has several fields with missing data. Choose a method to deal with missing data and justify your choice [2 marks]."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Step one is to just explore the data and to make simple observations about it, this starts with loading in the data and looking inside of it using pandas."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "             Row_ID  Household_ID       Vehicle  Calendar_Year    Model_Year  \\\n",
       "count  3.000000e+04  3.000000e+04  30000.000000   30000.000000  30000.000000   \n",
       "mean   5.265666e+06  3.415164e+06      1.869133    2006.118667   1999.505933   \n",
       "std    3.034083e+06  1.942246e+06      1.150848       0.804947      5.048889   \n",
       "min    1.160000e+03  6.830000e+02      1.000000    2005.000000   1981.000000   \n",
       "25%    2.670346e+06  1.917093e+06      1.000000    2005.000000   1997.000000   \n",
       "50%    5.255290e+06  3.654160e+06      2.000000    2006.000000   2000.000000   \n",
       "75%    7.858568e+06  4.992703e+06      2.000000    2007.000000   2003.000000   \n",
       "max    1.051789e+07  6.484624e+06     17.000000    2007.000000   2008.000000   \n",
       "\n",
       "             OrdCat          Var1          Var2          Var3          Var4  \\\n",
       "count  29981.000000  30000.000000  30000.000000  30000.000000  30000.000000   \n",
       "mean       3.575798     -0.006781     -0.063997     -0.025057     -0.053640   \n",
       "std        1.159251      0.976902      0.960495      1.016570      0.959304   \n",
       "min        1.000000     -2.578222     -2.441519     -2.744055     -2.457475   \n",
       "25%        2.000000     -0.665897     -0.816152     -0.869687     -0.783019   \n",
       "50%        4.000000     -0.320393     -0.124506     -0.221758     -0.106471   \n",
       "75%        4.000000      0.442930      0.480684      0.726996      0.485509   \n",
       "max        7.000000      4.540768      6.307799      5.563325      6.100857   \n",
       "\n",
       "               Var5          Var6          Var7          Var8        NVVar1  \\\n",
       "count  30000.000000  30000.000000  30000.000000  30000.000000  30000.000000   \n",
       "mean       0.012070     -0.044910     -0.024929     -0.064832      0.032385   \n",
       "std        0.986830      0.971651      1.000754      0.979235      1.063700   \n",
       "min       -3.350344     -2.300626     -2.262411     -1.882994     -0.231530   \n",
       "25%       -0.662235     -0.688765     -0.898486     -0.646298     -0.231530   \n",
       "50%       -0.115098     -0.241936     -0.468419     -0.269656     -0.231530   \n",
       "75%        0.550982      0.500830      0.870931      0.327962     -0.231530   \n",
       "max        3.869488      4.584289      4.127148     33.457737      6.627110   \n",
       "\n",
       "             NVVar2        NVVar3        NVVar4  Claim_Amount  \n",
       "count  30000.000000  30000.000000  30000.000000  30000.000000  \n",
       "mean       0.071626      0.078748      0.041319     54.862481  \n",
       "std        1.148950      1.144502      1.073275    244.230985  \n",
       "min       -0.266117     -0.272337     -0.251419      0.000000  \n",
       "25%       -0.266117     -0.272337     -0.251419      0.000000  \n",
       "50%       -0.266117     -0.272337     -0.251419      0.000000  \n",
       "75%       -0.266117     -0.272337     -0.251419      5.905321  \n",
       "max        8.883081      8.691144      6.388802  11440.750000  "
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Row_ID</th>\n      <th>Household_ID</th>\n      <th>Vehicle</th>\n      <th>Calendar_Year</th>\n      <th>Model_Year</th>\n      <th>OrdCat</th>\n      <th>Var1</th>\n      <th>Var2</th>\n      <th>Var3</th>\n      <th>Var4</th>\n      <th>Var5</th>\n      <th>Var6</th>\n      <th>Var7</th>\n      <th>Var8</th>\n      <th>NVVar1</th>\n      <th>NVVar2</th>\n      <th>NVVar3</th>\n      <th>NVVar4</th>\n      <th>Claim_Amount</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>3.000000e+04</td>\n      <td>3.000000e+04</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>29981.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n      <td>30000.000000</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>5.265666e+06</td>\n      <td>3.415164e+06</td>\n      <td>1.869133</td>\n      <td>2006.118667</td>\n      <td>1999.505933</td>\n      <td>3.575798</td>\n      <td>-0.006781</td>\n      <td>-0.063997</td>\n      <td>-0.025057</td>\n      <td>-0.053640</td>\n      <td>0.012070</td>\n      <td>-0.044910</td>\n      <td>-0.024929</td>\n      <td>-0.064832</td>\n      <td>0.032385</td>\n      <td>0.071626</td>\n      <td>0.078748</td>\n      <td>0.041319</td>\n      <td>54.862481</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>3.034083e+06</td>\n      <td>1.942246e+06</td>\n      <td>1.150848</td>\n      <td>0.804947</td>\n      <td>5.048889</td>\n      <td>1.159251</td>\n      <td>0.976902</td>\n      <td>0.960495</td>\n      <td>1.016570</td>\n      <td>0.959304</td>\n      <td>0.986830</td>\n      <td>0.971651</td>\n      <td>1.000754</td>\n      <td>0.979235</td>\n      <td>1.063700</td>\n      <td>1.148950</td>\n      <td>1.144502</td>\n      <td>1.073275</td>\n      <td>244.230985</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>1.160000e+03</td>\n      <td>6.830000e+02</td>\n      <td>1.000000</td>\n      <td>2005.000000</td>\n      <td>1981.000000</td>\n      <td>1.000000</td>\n      <td>-2.578222</td>\n      <td>-2.441519</td>\n      <td>-2.744055</td>\n      <td>-2.457475</td>\n      <td>-3.350344</td>\n      <td>-2.300626</td>\n      <td>-2.262411</td>\n      <td>-1.882994</td>\n      <td>-0.231530</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>2.670346e+06</td>\n      <td>1.917093e+06</td>\n      <td>1.000000</td>\n      <td>2005.000000</td>\n      <td>1997.000000</td>\n      <td>2.000000</td>\n      <td>-0.665897</td>\n      <td>-0.816152</td>\n      <td>-0.869687</td>\n      <td>-0.783019</td>\n      <td>-0.662235</td>\n      <td>-0.688765</td>\n      <td>-0.898486</td>\n      <td>-0.646298</td>\n      <td>-0.231530</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>5.255290e+06</td>\n      <td>3.654160e+06</td>\n      <td>2.000000</td>\n      <td>2006.000000</td>\n      <td>2000.000000</td>\n      <td>4.000000</td>\n      <td>-0.320393</td>\n      <td>-0.124506</td>\n      <td>-0.221758</td>\n      <td>-0.106471</td>\n      <td>-0.115098</td>\n      <td>-0.241936</td>\n      <td>-0.468419</td>\n      <td>-0.269656</td>\n      <td>-0.231530</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>7.858568e+06</td>\n      <td>4.992703e+06</td>\n      <td>2.000000</td>\n      <td>2007.000000</td>\n      <td>2003.000000</td>\n      <td>4.000000</td>\n      <td>0.442930</td>\n      <td>0.480684</td>\n      <td>0.726996</td>\n      <td>0.485509</td>\n      <td>0.550982</td>\n      <td>0.500830</td>\n      <td>0.870931</td>\n      <td>0.327962</td>\n      <td>-0.231530</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>5.905321</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>1.051789e+07</td>\n      <td>6.484624e+06</td>\n      <td>17.000000</td>\n      <td>2007.000000</td>\n      <td>2008.000000</td>\n      <td>7.000000</td>\n      <td>4.540768</td>\n      <td>6.307799</td>\n      <td>5.563325</td>\n      <td>6.100857</td>\n      <td>3.869488</td>\n      <td>4.584289</td>\n      <td>4.127148</td>\n      <td>33.457737</td>\n      <td>6.627110</td>\n      <td>8.883081</td>\n      <td>8.691144</td>\n      <td>6.388802</td>\n      <td>11440.750000</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 1
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import numpy as np\n",
    "#Will convert '?' to NaN as well\n",
    "claim_data = pd.read_csv('./data/train.csv', encoding= 'unicode_escape', na_values='?')\n",
    "claim_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 30000 entries, 0 to 29999\nData columns (total 35 columns):\n #   Column          Non-Null Count  Dtype  \n---  ------          --------------  -----  \n 0   Row_ID          30000 non-null  float64\n 1   Household_ID    30000 non-null  float64\n 2   Vehicle         30000 non-null  float64\n 3   Calendar_Year   30000 non-null  float64\n 4   Model_Year      30000 non-null  float64\n 5   Blind_Make      29985 non-null  object \n 6   Blind_Model     29985 non-null  object \n 7   Blind_Submodel  29985 non-null  object \n 8   Cat1            29950 non-null  object \n 9   Cat2            19409 non-null  object \n 10  Cat3            29989 non-null  object \n 11  Cat4            17015 non-null  object \n 12  Cat5            17002 non-null  object \n 13  Cat6            29950 non-null  object \n 14  Cat7            13520 non-null  object \n 15  Cat8            29998 non-null  object \n 16  Cat9            30000 non-null  object \n 17  Cat10           29990 non-null  object \n 18  Cat11           29942 non-null  object \n 19  Cat12           29948 non-null  object \n 20  OrdCat          29981 non-null  float64\n 21  Var1            30000 non-null  float64\n 22  Var2            30000 non-null  float64\n 23  Var3            30000 non-null  float64\n 24  Var4            30000 non-null  float64\n 25  Var5            30000 non-null  float64\n 26  Var6            30000 non-null  float64\n 27  Var7            30000 non-null  float64\n 28  Var8            30000 non-null  float64\n 29  NVCat           30000 non-null  object \n 30  NVVar1          30000 non-null  float64\n 31  NVVar2          30000 non-null  float64\n 32  NVVar3          30000 non-null  float64\n 33  NVVar4          30000 non-null  float64\n 34  Claim_Amount    30000 non-null  float64\ndtypes: float64(19), object(16)\nmemory usage: 8.0+ MB\n"
     ]
    }
   ],
   "source": [
    "for col in ['Row_ID', 'Household_ID', 'Vehicle', 'Calendar_Year', 'Model_Year']:\n",
    "    claim_data[col] = claim_data[col].astype('float64')\n",
    "claim_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "          Row_ID  Household_ID  Vehicle  Calendar_Year  Model_Year Blind_Make  \\\n",
       "29174  1476518.0      848179.0      1.0         2007.0      2001.0          Y   \n",
       "6400   2566090.0     1878410.0      3.0         2007.0      2000.0         BH   \n",
       "26074  8416163.0     5475561.0      2.0         2007.0      1998.0         AU   \n",
       "17934  9940541.0     6296841.0      1.0         2007.0      1998.0         AJ   \n",
       "1857   6469166.0     4206858.0      1.0         2007.0      2005.0         AF   \n",
       "\n",
       "      Blind_Model Blind_Submodel Cat1 Cat2  ...      Var5      Var6      Var7  \\\n",
       "29174        Y.32         Y.32.0    B    C  ...  0.580718 -0.300421 -0.972211   \n",
       "6400        BH.18        BH.18.5    B    C  ... -0.198358 -0.714499 -0.861623   \n",
       "26074       AU.54        AU.54.2    G  NaN  ... -0.073468  0.473927  0.354851   \n",
       "17934       AJ.14        AJ.14.3    H  NaN  ...  1.234903  1.243596  2.210280   \n",
       "1857        AF.24        AF.24.5    A    C  ... -0.436244 -0.106249 -0.738747   \n",
       "\n",
       "           Var8 NVCat   NVVar1    NVVar2    NVVar3    NVVar4 Claim_Amount  \n",
       "29174 -0.309366     O -0.23153 -0.266117 -0.272337 -0.251419    17.962280  \n",
       "6400  -0.559111     O -0.23153 -0.266117 -0.272337 -0.251419     8.513937  \n",
       "26074  0.021441     M -0.23153 -0.266117 -0.272337 -0.251419     0.000000  \n",
       "17934 -0.397428     O -0.23153 -0.266117 -0.272337 -0.251419     0.000000  \n",
       "1857   1.021956     B -0.23153 -0.266117 -0.272337  4.175396     0.000000  \n",
       "\n",
       "[5 rows x 35 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Row_ID</th>\n      <th>Household_ID</th>\n      <th>Vehicle</th>\n      <th>Calendar_Year</th>\n      <th>Model_Year</th>\n      <th>Blind_Make</th>\n      <th>Blind_Model</th>\n      <th>Blind_Submodel</th>\n      <th>Cat1</th>\n      <th>Cat2</th>\n      <th>...</th>\n      <th>Var5</th>\n      <th>Var6</th>\n      <th>Var7</th>\n      <th>Var8</th>\n      <th>NVCat</th>\n      <th>NVVar1</th>\n      <th>NVVar2</th>\n      <th>NVVar3</th>\n      <th>NVVar4</th>\n      <th>Claim_Amount</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>29174</th>\n      <td>1476518.0</td>\n      <td>848179.0</td>\n      <td>1.0</td>\n      <td>2007.0</td>\n      <td>2001.0</td>\n      <td>Y</td>\n      <td>Y.32</td>\n      <td>Y.32.0</td>\n      <td>B</td>\n      <td>C</td>\n      <td>...</td>\n      <td>0.580718</td>\n      <td>-0.300421</td>\n      <td>-0.972211</td>\n      <td>-0.309366</td>\n      <td>O</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>17.962280</td>\n    </tr>\n    <tr>\n      <th>6400</th>\n      <td>2566090.0</td>\n      <td>1878410.0</td>\n      <td>3.0</td>\n      <td>2007.0</td>\n      <td>2000.0</td>\n      <td>BH</td>\n      <td>BH.18</td>\n      <td>BH.18.5</td>\n      <td>B</td>\n      <td>C</td>\n      <td>...</td>\n      <td>-0.198358</td>\n      <td>-0.714499</td>\n      <td>-0.861623</td>\n      <td>-0.559111</td>\n      <td>O</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>8.513937</td>\n    </tr>\n    <tr>\n      <th>26074</th>\n      <td>8416163.0</td>\n      <td>5475561.0</td>\n      <td>2.0</td>\n      <td>2007.0</td>\n      <td>1998.0</td>\n      <td>AU</td>\n      <td>AU.54</td>\n      <td>AU.54.2</td>\n      <td>G</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>-0.073468</td>\n      <td>0.473927</td>\n      <td>0.354851</td>\n      <td>0.021441</td>\n      <td>M</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>17934</th>\n      <td>9940541.0</td>\n      <td>6296841.0</td>\n      <td>1.0</td>\n      <td>2007.0</td>\n      <td>1998.0</td>\n      <td>AJ</td>\n      <td>AJ.14</td>\n      <td>AJ.14.3</td>\n      <td>H</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>1.234903</td>\n      <td>1.243596</td>\n      <td>2.210280</td>\n      <td>-0.397428</td>\n      <td>O</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>1857</th>\n      <td>6469166.0</td>\n      <td>4206858.0</td>\n      <td>1.0</td>\n      <td>2007.0</td>\n      <td>2005.0</td>\n      <td>AF</td>\n      <td>AF.24</td>\n      <td>AF.24.5</td>\n      <td>A</td>\n      <td>C</td>\n      <td>...</td>\n      <td>-0.436244</td>\n      <td>-0.106249</td>\n      <td>-0.738747</td>\n      <td>1.021956</td>\n      <td>B</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>4.175396</td>\n      <td>0.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 35 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "claim_data.sample(5)"
   ]
  },
  {
   "source": [
    "From studying the data the first thing needed is a method to deal with the missing data in the categorical data. All other data seems consistent."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "#First count how many NaN we have in the categorical data points, print out (col_name,amount,data count,%)\n",
    "def count_nulls():\n",
    "    count_null_array = []\n",
    "    row_count = claim_data.shape[0]\n",
    "    for i in range(1,13):\n",
    "        cat_string = \"Cat\" + str(i)\n",
    "        output = claim_data[cat_string].isnull().sum()\n",
    "        count_null_array.append((\"Cat\" + str(i), output, row_count, round((output/row_count)*100,2)))\n",
    "    #Do the named ones\n",
    "    output = claim_data[\"OrdCat\"].isnull().sum()\n",
    "    count_null_array.append((\"OrdCat\",output, row_count, round((output/row_count)*100,2)))\n",
    "\n",
    "    output = claim_data[\"NVCat\"].isnull().sum()\n",
    "    count_null_array.append((\"NVCat\", output, row_count, round((output/row_count)*100,2)))\n",
    "    return count_null_array\n",
    "count_nulls()"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 4,
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('Cat1', 50, 30000, 0.17),\n",
       " ('Cat2', 10591, 30000, 35.3),\n",
       " ('Cat3', 11, 30000, 0.04),\n",
       " ('Cat4', 12985, 30000, 43.28),\n",
       " ('Cat5', 12998, 30000, 43.33),\n",
       " ('Cat6', 50, 30000, 0.17),\n",
       " ('Cat7', 16480, 30000, 54.93),\n",
       " ('Cat8', 2, 30000, 0.01),\n",
       " ('Cat9', 0, 30000, 0.0),\n",
       " ('Cat10', 10, 30000, 0.03),\n",
       " ('Cat11', 58, 30000, 0.19),\n",
       " ('Cat12', 52, 30000, 0.17),\n",
       " ('OrdCat', 19, 30000, 0.06),\n",
       " ('NVCat', 0, 30000, 0.0)]"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ]
  },
  {
   "source": [
    "Here we can see two major different types of missing data. We have Cat1,3,6,8,10,11,12,OrdCat with very little data missing opposed to Cat2,4,5,7 with lots missing. This means it maybe appropite to deal with these two types of missing data in seperate ways. For the values with little missing data I am simply going to remove the data from the dataset. While I could maybe impute using a most common method or another I think just removing dirty data is better when it is so small. In a worse case where none of the missing data overlaps at most I will lose 252 rows of data, so less than 1%. I do not believe this will affect the training data enough to justify the imputation. Imputation is really only a best guess and is sub optimal for data. The columns with no missing values will obviously have nothing done to them. Below I will remove the rows with ? in the data for these columns. It is also worth noting that the model classes have 15 rows missing."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "#Remove the row if a ? is in the selected column, also add in the model with missing numbers\n",
    "columns_to_remove = [\"Cat1\",\"Cat3\",\"Cat6\",\"Cat8\",\"Cat10\",\"Cat11\",\"Cat12\",\"OrdCat\",\"Blind_Make\",\"Blind_Model\",\"Blind_Submodel\"]\n",
    "claim_data = claim_data.dropna(subset=columns_to_remove)\n",
    "claim_data = claim_data.reset_index(drop=True)\n",
    "print(claim_data)\n",
    "count_nulls()\n"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 5,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "           Row_ID  Household_ID  Vehicle  Calendar_Year  Model_Year  \\\n0       2718481.0     1933325.0      3.0         2006.0      1997.0   \n1       6717427.0     4297298.0      1.0         2006.0      1997.0   \n2       7218252.0     4708245.0      1.0         2006.0      2000.0   \n3       8048505.0     5136981.0      7.0         2007.0      2003.0   \n4       3209482.0     2151306.0      5.0         2006.0      2001.0   \n...           ...           ...      ...            ...         ...   \n29830  10424872.0     6456045.0      2.0         2007.0      2000.0   \n29831   6304361.0     4152505.0      4.0         2007.0      2002.0   \n29832    895720.0      596804.0      1.0         2005.0      1991.0   \n29833   3426228.0     2247109.0      1.0         2005.0      1992.0   \n29834   7357807.0     4773632.0      1.0         2006.0      1997.0   \n\n      Blind_Make Blind_Model Blind_Submodel Cat1 Cat2  ...      Var5  \\\n0              R        R.20         R.20.6    B  NaN  ... -1.054747   \n1             BF       BF.35        BF.35.1    F  NaN  ... -1.149901   \n2              M        M.16         M.16.1    B    C  ... -0.775231   \n3             BH       BH.22        BH.22.0    B    C  ... -0.751442   \n4             BW      BW.167       BW.167.0    I    C  ...  0.461775   \n...          ...         ...            ...  ...  ...  ...       ...   \n29830         BO       BO.32        BO.32.1    D    A  ...  1.425212   \n29831          Y         Y.9          Y.9.4    F    C  ... -0.608711   \n29832          X        X.45         X.45.2    B  NaN  ... -0.162675   \n29833          X        X.45         X.45.3    B  NaN  ...  0.051422   \n29834         BF       BF.10        BF.10.2    F  NaN  ... -0.055627   \n\n           Var6      Var7      Var8 NVCat   NVVar1    NVVar2    NVVar3  \\\n0     -1.164837 -0.812472 -0.581756     N -0.23153 -0.266117  2.715490   \n1     -1.299354 -1.193388 -1.323992     N -0.23153  2.783616 -0.272337   \n2     -1.467792 -0.947636 -0.914860     F -0.23153 -0.266117 -0.272337   \n3     -0.378792 -0.689596  0.730640     M -0.23153 -0.266117 -0.272337   \n4      1.451805  1.411586  0.977104     B -0.23153 -0.266117 -0.272337   \n...         ...       ...       ...   ...      ...       ...       ...   \n29830  0.907890  0.711192  0.194939     M -0.23153 -0.266117 -0.272337   \n29831 -1.134425 -1.193388 -0.844848     L -0.23153 -0.266117  4.209404   \n29832 -0.597528 -1.058225 -0.728890     N -0.23153 -0.266117  2.715490   \n29833 -0.578812 -1.058225 -0.656034     M -0.23153 -0.266117 -0.272337   \n29834 -0.829130 -1.193388 -0.906108     M -0.23153 -0.266117 -0.272337   \n\n         NVVar4 Claim_Amount  \n0     -0.251419   104.597500  \n1     -0.251419     0.000000  \n2     -0.251419     0.000000  \n3     -0.251419     0.000000  \n4     -0.251419     0.615937  \n...         ...          ...  \n29830 -0.251419     0.000000  \n29831 -0.251419   135.602700  \n29832 -0.251419    42.755250  \n29833 -0.251419     0.000000  \n29834 -0.251419     0.000000  \n\n[29835 rows x 35 columns]\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('Cat1', 0, 29835, 0.0),\n",
       " ('Cat2', 10510, 29835, 35.23),\n",
       " ('Cat3', 0, 29835, 0.0),\n",
       " ('Cat4', 12905, 29835, 43.25),\n",
       " ('Cat5', 12918, 29835, 43.3),\n",
       " ('Cat6', 0, 29835, 0.0),\n",
       " ('Cat7', 16376, 29835, 54.89),\n",
       " ('Cat8', 0, 29835, 0.0),\n",
       " ('Cat9', 0, 29835, 0.0),\n",
       " ('Cat10', 0, 29835, 0.0),\n",
       " ('Cat11', 0, 29835, 0.0),\n",
       " ('Cat12', 0, 29835, 0.0),\n",
       " ('OrdCat', 0, 29835, 0.0),\n",
       " ('NVCat', 0, 29835, 0.0)]"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ]
  },
  {
   "source": [
    "As you can now see there has been 165 rows lost, however we now have most columns with no missing data. A small price to pay for this much cleaner data. Looking at the classes with lots of missing data, I'd like to explore what is missing and see what the columns could have been. To do this I will do a value count again but print in full"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Cat2\nB      555\nA     5188\nC    13582\nName: Cat2, dtype: int64\nCat4\nB      850\nC     3255\nA    12825\nName: Cat4, dtype: int64\nCat5\nB      163\nC     1617\nA    15137\nName: Cat5, dtype: int64\nCat7\nD      274\nB      491\nA     2184\nC    10510\nName: Cat7, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "columns_to_inv = [\"Cat2\", \"Cat4\", \"Cat5\", \"Cat7\"]\n",
    "for inv_str in columns_to_inv:\n",
    "    print(inv_str)\n",
    "    print(claim_data[inv_str].value_counts(ascending=True))"
   ]
  },
  {
   "source": [
    "The data here shows that each one with missing data has a very obvious most common answer. All have a large modal favourite in the feature, such as C in Cat7. This makes them a candidate for modal imputation. Another method is to use a KNN classifier to impute these missing values. This method is more effective as it considers the correlation between other types of features to predict the most likely choice for the variable. This does make the assumption that the missing values are dependent on the others however, but no method to solve this is perfect. "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "52709\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "#Confirm we have NA data\n",
    "print(claim_data.isna().sum().sum())\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "#Generate our one hot encoding in the dataframe\n",
    "cat_variables = claim_data[[\"Cat1\",\"Cat2\",\"Cat3\",\"Cat4\",\"Cat5\",\"Cat6\",\"Cat7\",\"Cat8\",\"Cat9\",\"Cat10\",\"Cat11\",\"Cat12\",\"OrdCat\",\"NVCat\"]]\n",
    "cat_dummies = pd.get_dummies(cat_variables)\n",
    "\n",
    "claim_data_ohe = pd.concat([claim_data.drop(cat_variables, axis=1), cat_dummies], axis=1)\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "#Too much data to onehotencode and package issues stopped me from hashing these - Not ideal but has to be done\n",
    "#Also remove the target variable\n",
    "claim_data_ohe = claim_data_ohe.drop([\"Blind_Make\",\"Blind_Model\", \"Blind_Submodel\", \"Claim_Amount\"], axis=1)\n",
    "\n",
    "#Normalise the data to between 0 and 1\n",
    "claim_data_ohe = pd.DataFrame(scaler.fit_transform(claim_data_ohe), columns= claim_data_ohe.columns)\n",
    "\n",
    "#Find with KNN\n",
    "imputer = KNNImputer(n_neighbors=7)\n",
    "\n",
    "claim_data_imputed = pd.DataFrame(imputer.fit_transform(claim_data_ohe), columns= claim_data_ohe.columns)\n",
    "\n",
    "#Reverse one hot encoding\n",
    "for cat in columns_to_inv:\n",
    "    new_column = pd.DataFrame(claim_data_imputed.filter(regex=\"^\" + cat + \"_\").idxmax(axis=1), columns=[cat])\n",
    "    regex_string = cat + \"_\"\n",
    "    claim_data[cat] = new_column[cat].str.replace(regex_string,'')\n",
    "#Confirm we have no nulls left in our dataset\n",
    "claim_data.isnull().sum().sum()"
   ]
  },
  {
   "source": [
    "claim_data is now a full dataset with no null entries. We have lost around 166 entries and imputed the rest that were null using a KNN classifier. The code above changes the data into OneHotEncoded in the dataframe, then normalises the data between 0 and 1 to allow the algorithm to work. It uses the nan_euclidian method given by scipy to best estimate what the guess should be for the missing value. The code then adds the updated values to the dataset and resets the other preprocessing work done so we can select better methods for our later algorithms. Below we can now repeat our earlier value counts to see how the classifier has changed them."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "for inv_str in columns_to_inv:\n",
    "    print(inv_str)\n",
    "    print(claim_data[inv_str].value_counts(ascending=True))"
   ],
   "cell_type": "code",
   "metadata": {},
   "execution_count": 8,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Cat2\nB      555\nC    13582\nA    15698\nName: Cat2, dtype: int64\nCat4\nB      850\nC     3255\nA    25730\nName: Cat4, dtype: int64\nCat5\nB      163\nC     1617\nA    28055\nName: Cat5, dtype: int64\nCat7\nD      274\nB      491\nC    10510\nA    18560\nName: Cat7, dtype: int64\n"
     ]
    }
   ]
  },
  {
   "source": [
    "For Cat4 and Cat 6 it has just always selected the modal answer, however Cat2 and Cat7 have provided a split between the variables. These are hopefully more accurate representations of what would have been there if the data was captured."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Question 1 - Data Preprocessing - Part B\n",
    "\n",
    "Convert categorical values to a suitable representation. Notice that there are many categorical variables in the dataset. If you use all the categorical variables you will end up with a large feature space. Feel free to ignore categorical variables that will increase your feature space considerably but use at least five categorical variables [2 marks]."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "For dealing with the categorical data I have decided on using OneHotEncoding for it. I will not include the ordered categories in this list as they have a natural ordering to them and are already numerical. I will also choose to totally ignore and drop the Blind_Model and Blind_Submodel. These are such variable data points and the model will likely not have seen the data ever before when a new claim is made due to the variety in these subsets. I will keep the make in as these are slightly less varied. This will create a very large matrix of possiblitys but it shouldn't cause any issues. For my numerical data I will use normilisation. This will work well due to the means being around 0 already and a standard deviation of 1 in place which the standard scaler in the lab class did. \n",
    "\n",
    "Below I will create a column transformer to call later on the data that will do all of this work"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "attributes_num = [\"Var1\", \"Var2\",\"Var3\",\"Var4\",\"Var5\",\"Var6\",\"Var7\",\"Var8\",\"NVVar1\",\"NVVar2\",\"NVVar3\",\"NVVar4\"]\n",
    "attributes_cat = [\"Blind_Make\", \"Cat1\",\"Cat2\",\"Cat3\",\"Cat4\",\"Cat5\",\"Cat6\",\"Cat7\",\"Cat8\",\"Cat9\",\"Cat10\",\"Cat11\",\"Cat12\",\"NVCat\"]\n",
    "attributes_other = [\"Household_ID\", \"Vehicle\", \"Calendar_Year\", \"Model_Year\", \"OrdCat\"]\n",
    "\n",
    "#Below I drop some columns, these are used in the actual transformer as opposed to all the columns which are above\n",
    "attributes_num_used = [\"Var1\", \"Var2\",\"Var3\",\"Var4\",\"Var5\",\"Var6\",\"Var7\",\"Var8\",\"NVVar1\",\"NVVar2\",\"NVVar3\",\"NVVar4\"]\n",
    "attributes_cat_used = [\"Cat1\",\"Cat3\",\"Cat5\",\"Cat6\",\"Cat7\",\"Cat8\",\"Cat9\",\"NVCat\"]\n",
    "\n",
    "full_transform = ColumnTransformer([\n",
    "    (\"num\", Normalizer(), attributes_num_used),\n",
    "    (\"cat\", OneHotEncoder(), attributes_cat_used),\n",
    "])\n",
    "\n",
    "#Transformer without numerical for the tree models\n",
    "tree_transformer = ColumnTransformer([\n",
    "    (\"cat\", OneHotEncoder(), attributes_cat_used)\n",
    "],remainder='passthrough')\n",
    "\n",
    "#Drop these pieces of data, they are not needed\n",
    "claim_data = claim_data.drop([\"Blind_Model\", \"Blind_Submodel\"], axis=1)"
   ]
  },
  {
   "source": [
    "Question 1 - Data Preprocessing - Part C\n",
    "\n",
    "The data is highly imbalanced: more records contain zero claims than not.  When designing your predictive model, you need to account for this [3 marks].\n",
    "\n",
    "Lets first of all observe the data imbalance by a simple bar chart comparing the two types of target variable, 0 claim and < 0 claim."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "20877\n8958\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<BarContainer object of 2 artists>"
      ]
     },
     "metadata": {},
     "execution_count": 10
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n<svg height=\"248.518125pt\" version=\"1.1\" viewBox=\"0 0 388.0125 248.518125\" width=\"388.0125pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n <metadata>\r\n  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n   <cc:Work>\r\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n    <dc:date>2020-10-29T18:11:06.791129</dc:date>\r\n    <dc:format>image/svg+xml</dc:format>\r\n    <dc:creator>\r\n     <cc:Agent>\r\n      <dc:title>Matplotlib v3.3.2, https://matplotlib.org/</dc:title>\r\n     </cc:Agent>\r\n    </dc:creator>\r\n   </cc:Work>\r\n  </rdf:RDF>\r\n </metadata>\r\n <defs>\r\n  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n </defs>\r\n <g id=\"figure_1\">\r\n  <g id=\"patch_1\">\r\n   <path d=\"M 0 248.518125 \r\nL 388.0125 248.518125 \r\nL 388.0125 0 \r\nL 0 0 \r\nz\r\n\" style=\"fill:none;\"/>\r\n  </g>\r\n  <g id=\"axes_1\">\r\n   <g id=\"patch_2\">\r\n    <path d=\"M 46.0125 224.64 \r\nL 380.8125 224.64 \r\nL 380.8125 7.2 \r\nL 46.0125 7.2 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n   </g>\r\n   <g id=\"patch_3\">\r\n    <path clip-path=\"url(#pec2dc0d3f9)\" d=\"M 61.230682 224.64 \r\nL 196.503409 224.64 \r\nL 196.503409 17.554286 \r\nL 61.230682 17.554286 \r\nz\r\n\" style=\"fill:#1f77b4;\"/>\r\n   </g>\r\n   <g id=\"patch_4\">\r\n    <path clip-path=\"url(#pec2dc0d3f9)\" d=\"M 230.321591 224.64 \r\nL 365.594318 224.64 \r\nL 365.594318 135.782701 \r\nL 230.321591 135.782701 \r\nz\r\n\" style=\"fill:#1f77b4;\"/>\r\n   </g>\r\n   <g id=\"matplotlib.axis_1\">\r\n    <g id=\"xtick_1\">\r\n     <g id=\"line2d_1\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL 0 3.5 \r\n\" id=\"mf51df1484d\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"128.867045\" xlink:href=\"#mf51df1484d\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_1\">\r\n      <!-- Zero Count -->\r\n      <g transform=\"translate(100.925639 239.238437)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 5.609375 72.90625 \r\nL 62.890625 72.90625 \r\nL 62.890625 65.375 \r\nL 16.796875 8.296875 \r\nL 64.015625 8.296875 \r\nL 64.015625 0 \r\nL 4.5 0 \r\nL 4.5 7.515625 \r\nL 50.59375 64.59375 \r\nL 5.609375 64.59375 \r\nz\r\n\" id=\"DejaVuSans-90\"/>\r\n        <path d=\"M 56.203125 29.59375 \r\nL 56.203125 25.203125 \r\nL 14.890625 25.203125 \r\nQ 15.484375 15.921875 20.484375 11.0625 \r\nQ 25.484375 6.203125 34.421875 6.203125 \r\nQ 39.59375 6.203125 44.453125 7.46875 \r\nQ 49.3125 8.734375 54.109375 11.28125 \r\nL 54.109375 2.78125 \r\nQ 49.265625 0.734375 44.1875 -0.34375 \r\nQ 39.109375 -1.421875 33.890625 -1.421875 \r\nQ 20.796875 -1.421875 13.15625 6.1875 \r\nQ 5.515625 13.8125 5.515625 26.8125 \r\nQ 5.515625 40.234375 12.765625 48.109375 \r\nQ 20.015625 56 32.328125 56 \r\nQ 43.359375 56 49.78125 48.890625 \r\nQ 56.203125 41.796875 56.203125 29.59375 \r\nz\r\nM 47.21875 32.234375 \r\nQ 47.125 39.59375 43.09375 43.984375 \r\nQ 39.0625 48.390625 32.421875 48.390625 \r\nQ 24.90625 48.390625 20.390625 44.140625 \r\nQ 15.875 39.890625 15.1875 32.171875 \r\nz\r\n\" id=\"DejaVuSans-101\"/>\r\n        <path d=\"M 41.109375 46.296875 \r\nQ 39.59375 47.171875 37.8125 47.578125 \r\nQ 36.03125 48 33.890625 48 \r\nQ 26.265625 48 22.1875 43.046875 \r\nQ 18.109375 38.09375 18.109375 28.8125 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 20.953125 51.171875 25.484375 53.578125 \r\nQ 30.03125 56 36.53125 56 \r\nQ 37.453125 56 38.578125 55.875 \r\nQ 39.703125 55.765625 41.0625 55.515625 \r\nz\r\n\" id=\"DejaVuSans-114\"/>\r\n        <path d=\"M 30.609375 48.390625 \r\nQ 23.390625 48.390625 19.1875 42.75 \r\nQ 14.984375 37.109375 14.984375 27.296875 \r\nQ 14.984375 17.484375 19.15625 11.84375 \r\nQ 23.34375 6.203125 30.609375 6.203125 \r\nQ 37.796875 6.203125 41.984375 11.859375 \r\nQ 46.1875 17.53125 46.1875 27.296875 \r\nQ 46.1875 37.015625 41.984375 42.703125 \r\nQ 37.796875 48.390625 30.609375 48.390625 \r\nz\r\nM 30.609375 56 \r\nQ 42.328125 56 49.015625 48.375 \r\nQ 55.71875 40.765625 55.71875 27.296875 \r\nQ 55.71875 13.875 49.015625 6.21875 \r\nQ 42.328125 -1.421875 30.609375 -1.421875 \r\nQ 18.84375 -1.421875 12.171875 6.21875 \r\nQ 5.515625 13.875 5.515625 27.296875 \r\nQ 5.515625 40.765625 12.171875 48.375 \r\nQ 18.84375 56 30.609375 56 \r\nz\r\n\" id=\"DejaVuSans-111\"/>\r\n        <path id=\"DejaVuSans-32\"/>\r\n        <path d=\"M 64.40625 67.28125 \r\nL 64.40625 56.890625 \r\nQ 59.421875 61.53125 53.78125 63.8125 \r\nQ 48.140625 66.109375 41.796875 66.109375 \r\nQ 29.296875 66.109375 22.65625 58.46875 \r\nQ 16.015625 50.828125 16.015625 36.375 \r\nQ 16.015625 21.96875 22.65625 14.328125 \r\nQ 29.296875 6.6875 41.796875 6.6875 \r\nQ 48.140625 6.6875 53.78125 8.984375 \r\nQ 59.421875 11.28125 64.40625 15.921875 \r\nL 64.40625 5.609375 \r\nQ 59.234375 2.09375 53.4375 0.328125 \r\nQ 47.65625 -1.421875 41.21875 -1.421875 \r\nQ 24.65625 -1.421875 15.125 8.703125 \r\nQ 5.609375 18.84375 5.609375 36.375 \r\nQ 5.609375 53.953125 15.125 64.078125 \r\nQ 24.65625 74.21875 41.21875 74.21875 \r\nQ 47.75 74.21875 53.53125 72.484375 \r\nQ 59.328125 70.75 64.40625 67.28125 \r\nz\r\n\" id=\"DejaVuSans-67\"/>\r\n        <path d=\"M 8.5 21.578125 \r\nL 8.5 54.6875 \r\nL 17.484375 54.6875 \r\nL 17.484375 21.921875 \r\nQ 17.484375 14.15625 20.5 10.265625 \r\nQ 23.53125 6.390625 29.59375 6.390625 \r\nQ 36.859375 6.390625 41.078125 11.03125 \r\nQ 45.3125 15.671875 45.3125 23.6875 \r\nL 45.3125 54.6875 \r\nL 54.296875 54.6875 \r\nL 54.296875 0 \r\nL 45.3125 0 \r\nL 45.3125 8.40625 \r\nQ 42.046875 3.421875 37.71875 1 \r\nQ 33.40625 -1.421875 27.6875 -1.421875 \r\nQ 18.265625 -1.421875 13.375 4.4375 \r\nQ 8.5 10.296875 8.5 21.578125 \r\nz\r\nM 31.109375 56 \r\nz\r\n\" id=\"DejaVuSans-117\"/>\r\n        <path d=\"M 54.890625 33.015625 \r\nL 54.890625 0 \r\nL 45.90625 0 \r\nL 45.90625 32.71875 \r\nQ 45.90625 40.484375 42.875 44.328125 \r\nQ 39.84375 48.1875 33.796875 48.1875 \r\nQ 26.515625 48.1875 22.3125 43.546875 \r\nQ 18.109375 38.921875 18.109375 30.90625 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 21.34375 51.125 25.703125 53.5625 \r\nQ 30.078125 56 35.796875 56 \r\nQ 45.21875 56 50.046875 50.171875 \r\nQ 54.890625 44.34375 54.890625 33.015625 \r\nz\r\n\" id=\"DejaVuSans-110\"/>\r\n        <path d=\"M 18.3125 70.21875 \r\nL 18.3125 54.6875 \r\nL 36.8125 54.6875 \r\nL 36.8125 47.703125 \r\nL 18.3125 47.703125 \r\nL 18.3125 18.015625 \r\nQ 18.3125 11.328125 20.140625 9.421875 \r\nQ 21.96875 7.515625 27.59375 7.515625 \r\nL 36.8125 7.515625 \r\nL 36.8125 0 \r\nL 27.59375 0 \r\nQ 17.1875 0 13.234375 3.875 \r\nQ 9.28125 7.765625 9.28125 18.015625 \r\nL 9.28125 47.703125 \r\nL 2.6875 47.703125 \r\nL 2.6875 54.6875 \r\nL 9.28125 54.6875 \r\nL 9.28125 70.21875 \r\nz\r\n\" id=\"DejaVuSans-116\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-90\"/>\r\n       <use x=\"68.505859\" xlink:href=\"#DejaVuSans-101\"/>\r\n       <use x=\"130.029297\" xlink:href=\"#DejaVuSans-114\"/>\r\n       <use x=\"168.892578\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"230.074219\" xlink:href=\"#DejaVuSans-32\"/>\r\n       <use x=\"261.861328\" xlink:href=\"#DejaVuSans-67\"/>\r\n       <use x=\"331.685547\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"392.867188\" xlink:href=\"#DejaVuSans-117\"/>\r\n       <use x=\"456.246094\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"519.625\" xlink:href=\"#DejaVuSans-116\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_2\">\r\n     <g id=\"line2d_2\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"297.957955\" xlink:href=\"#mf51df1484d\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_2\">\r\n      <!-- Non-Zero Count -->\r\n      <g transform=\"translate(258.243892 239.238437)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 9.8125 72.90625 \r\nL 23.09375 72.90625 \r\nL 55.421875 11.921875 \r\nL 55.421875 72.90625 \r\nL 64.984375 72.90625 \r\nL 64.984375 0 \r\nL 51.703125 0 \r\nL 19.390625 60.984375 \r\nL 19.390625 0 \r\nL 9.8125 0 \r\nz\r\n\" id=\"DejaVuSans-78\"/>\r\n        <path d=\"M 4.890625 31.390625 \r\nL 31.203125 31.390625 \r\nL 31.203125 23.390625 \r\nL 4.890625 23.390625 \r\nz\r\n\" id=\"DejaVuSans-45\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-78\"/>\r\n       <use x=\"74.804688\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"135.986328\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"199.365234\" xlink:href=\"#DejaVuSans-45\"/>\r\n       <use x=\"235.449219\" xlink:href=\"#DejaVuSans-90\"/>\r\n       <use x=\"303.955078\" xlink:href=\"#DejaVuSans-101\"/>\r\n       <use x=\"365.478516\" xlink:href=\"#DejaVuSans-114\"/>\r\n       <use x=\"404.341797\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"465.523438\" xlink:href=\"#DejaVuSans-32\"/>\r\n       <use x=\"497.310547\" xlink:href=\"#DejaVuSans-67\"/>\r\n       <use x=\"567.134766\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"628.316406\" xlink:href=\"#DejaVuSans-117\"/>\r\n       <use x=\"691.695312\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"755.074219\" xlink:href=\"#DejaVuSans-116\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"matplotlib.axis_2\">\r\n    <g id=\"ytick_1\">\r\n     <g id=\"line2d_3\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL -3.5 0 \r\n\" id=\"m32fd2772b2\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_3\">\r\n      <!-- 0 -->\r\n      <g transform=\"translate(32.65 228.439219)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 31.78125 66.40625 \r\nQ 24.171875 66.40625 20.328125 58.90625 \r\nQ 16.5 51.421875 16.5 36.375 \r\nQ 16.5 21.390625 20.328125 13.890625 \r\nQ 24.171875 6.390625 31.78125 6.390625 \r\nQ 39.453125 6.390625 43.28125 13.890625 \r\nQ 47.125 21.390625 47.125 36.375 \r\nQ 47.125 51.421875 43.28125 58.90625 \r\nQ 39.453125 66.40625 31.78125 66.40625 \r\nz\r\nM 31.78125 74.21875 \r\nQ 44.046875 74.21875 50.515625 64.515625 \r\nQ 56.984375 54.828125 56.984375 36.375 \r\nQ 56.984375 17.96875 50.515625 8.265625 \r\nQ 44.046875 -1.421875 31.78125 -1.421875 \r\nQ 19.53125 -1.421875 13.0625 8.265625 \r\nQ 6.59375 17.96875 6.59375 36.375 \r\nQ 6.59375 54.828125 13.0625 64.515625 \r\nQ 19.53125 74.21875 31.78125 74.21875 \r\nz\r\n\" id=\"DejaVuSans-48\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_2\">\r\n     <g id=\"line2d_4\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"199.841692\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_4\">\r\n      <!-- 2500 -->\r\n      <g transform=\"translate(13.5625 203.64091)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 19.1875 8.296875 \r\nL 53.609375 8.296875 \r\nL 53.609375 0 \r\nL 7.328125 0 \r\nL 7.328125 8.296875 \r\nQ 12.9375 14.109375 22.625 23.890625 \r\nQ 32.328125 33.6875 34.8125 36.53125 \r\nQ 39.546875 41.84375 41.421875 45.53125 \r\nQ 43.3125 49.21875 43.3125 52.78125 \r\nQ 43.3125 58.59375 39.234375 62.25 \r\nQ 35.15625 65.921875 28.609375 65.921875 \r\nQ 23.96875 65.921875 18.8125 64.3125 \r\nQ 13.671875 62.703125 7.8125 59.421875 \r\nL 7.8125 69.390625 \r\nQ 13.765625 71.78125 18.9375 73 \r\nQ 24.125 74.21875 28.421875 74.21875 \r\nQ 39.75 74.21875 46.484375 68.546875 \r\nQ 53.21875 62.890625 53.21875 53.421875 \r\nQ 53.21875 48.921875 51.53125 44.890625 \r\nQ 49.859375 40.875 45.40625 35.40625 \r\nQ 44.1875 33.984375 37.640625 27.21875 \r\nQ 31.109375 20.453125 19.1875 8.296875 \r\nz\r\n\" id=\"DejaVuSans-50\"/>\r\n        <path d=\"M 10.796875 72.90625 \r\nL 49.515625 72.90625 \r\nL 49.515625 64.59375 \r\nL 19.828125 64.59375 \r\nL 19.828125 46.734375 \r\nQ 21.96875 47.46875 24.109375 47.828125 \r\nQ 26.265625 48.1875 28.421875 48.1875 \r\nQ 40.625 48.1875 47.75 41.5 \r\nQ 54.890625 34.8125 54.890625 23.390625 \r\nQ 54.890625 11.625 47.5625 5.09375 \r\nQ 40.234375 -1.421875 26.90625 -1.421875 \r\nQ 22.3125 -1.421875 17.546875 -0.640625 \r\nQ 12.796875 0.140625 7.71875 1.703125 \r\nL 7.71875 11.625 \r\nQ 12.109375 9.234375 16.796875 8.0625 \r\nQ 21.484375 6.890625 26.703125 6.890625 \r\nQ 35.15625 6.890625 40.078125 11.328125 \r\nQ 45.015625 15.765625 45.015625 23.390625 \r\nQ 45.015625 31 40.078125 35.4375 \r\nQ 35.15625 39.890625 26.703125 39.890625 \r\nQ 22.75 39.890625 18.8125 39.015625 \r\nQ 14.890625 38.140625 10.796875 36.28125 \r\nz\r\n\" id=\"DejaVuSans-53\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_3\">\r\n     <g id=\"line2d_5\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"175.043383\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_5\">\r\n      <!-- 5000 -->\r\n      <g transform=\"translate(13.5625 178.842602)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_4\">\r\n     <g id=\"line2d_6\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"150.245075\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_6\">\r\n      <!-- 7500 -->\r\n      <g transform=\"translate(13.5625 154.044293)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 8.203125 72.90625 \r\nL 55.078125 72.90625 \r\nL 55.078125 68.703125 \r\nL 28.609375 0 \r\nL 18.3125 0 \r\nL 43.21875 64.59375 \r\nL 8.203125 64.59375 \r\nz\r\n\" id=\"DejaVuSans-55\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-55\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_5\">\r\n     <g id=\"line2d_7\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"125.446766\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_7\">\r\n      <!-- 10000 -->\r\n      <g transform=\"translate(7.2 129.245985)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 12.40625 8.296875 \r\nL 28.515625 8.296875 \r\nL 28.515625 63.921875 \r\nL 10.984375 60.40625 \r\nL 10.984375 69.390625 \r\nL 28.421875 72.90625 \r\nL 38.28125 72.90625 \r\nL 38.28125 8.296875 \r\nL 54.390625 8.296875 \r\nL 54.390625 0 \r\nL 12.40625 0 \r\nz\r\n\" id=\"DejaVuSans-49\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_6\">\r\n     <g id=\"line2d_8\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"100.648458\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_8\">\r\n      <!-- 12500 -->\r\n      <g transform=\"translate(7.2 104.447676)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_7\">\r\n     <g id=\"line2d_9\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"75.850149\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_9\">\r\n      <!-- 15000 -->\r\n      <g transform=\"translate(7.2 79.649368)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_8\">\r\n     <g id=\"line2d_10\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"51.051841\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_10\">\r\n      <!-- 17500 -->\r\n      <g transform=\"translate(7.2 54.85106)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-55\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-53\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_9\">\r\n     <g id=\"line2d_11\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"46.0125\" xlink:href=\"#m32fd2772b2\" y=\"26.253532\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_11\">\r\n      <!-- 20000 -->\r\n      <g transform=\"translate(7.2 30.052751)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"254.492188\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"patch_5\">\r\n    <path d=\"M 46.0125 224.64 \r\nL 46.0125 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_6\">\r\n    <path d=\"M 380.8125 224.64 \r\nL 380.8125 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_7\">\r\n    <path d=\"M 46.0125 224.64 \r\nL 380.8125 224.64 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_8\">\r\n    <path d=\"M 46.0125 7.2 \r\nL 380.8125 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n  </g>\r\n </g>\r\n <defs>\r\n  <clipPath id=\"pec2dc0d3f9\">\r\n   <rect height=\"217.44\" width=\"334.8\" x=\"46.0125\" y=\"7.2\"/>\r\n  </clipPath>\r\n </defs>\r\n</svg>\r\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAWZElEQVR4nO3df5TddX3n8eerQahbi4QypjSJG+rGuui2UWaRXavFZYWAexp0uwrblaCskSN0a1vbBtuzUC1HqlJdTi001kjYoyAWKSnGpjH+Pm00A6aBoGwGDEvSAFNDQWtNDb73j/sZ9+swM5nMTGYG83ycc8/93vf38/l8Pze5yet+f8x8U1VIko5sPzLbE5AkzT7DQJJkGEiSDANJEoaBJAk4arYnMFknnHBCLVmyZLanIUlPKXfcccffV1XfyPpTNgyWLFnCwMDAbE9Dkp5SkjwwWt3DRJIkw0CSZBhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJ4in8E8hTsWT1J2Z7Cpqjdl31ytmegjQr3DOQJBkGkiTDQJKEYSBJYgJhkGRxks8kuSfJjiS/2urHJ9mUZGd7nt/qSXJNksEk25O8qDPWytZ+Z5KVnfopSe5qfa5JksPxZiVJo5vInsEB4Deq6mTgNOCSJCcDq4HNVbUU2NxeA5wNLG2PVcC10AsP4HLgxcCpwOXDAdLavLHTb/nU35okaaIOGgZVtbeq7mzL3wS+CiwEVgDrWrN1wLlteQVwQ/VsAY5LciJwFrCpqvZV1aPAJmB5W3dsVW2pqgJu6IwlSZoBh3TOIMkS4IXAl4AFVbW3rXoIWNCWFwIPdrrtbrXx6rtHqY+2/VVJBpIMDA0NHcrUJUnjmHAYJHkGcAvwlqp6vLuufaOvaZ7bk1TVmqrqr6r+vr4n3cJTkjRJEwqDJE+jFwQfrqqPt/LD7RAP7fmRVt8DLO50X9Rq49UXjVKXJM2QiVxNFOCDwFer6g87q9YDw1cErQRu69QvaFcVnQY81g4nbQTOTDK/nTg+E9jY1j2e5LS2rQs6Y0mSZsBEfjfRS4DXAXcl2dZqbwOuAm5OchHwAPCatm4DcA4wCHwbeD1AVe1L8g5ga2v39qra15bfDFwPPB34ZHtIkmbIQcOgqr4IjHXd/xmjtC/gkjHGWgusHaU+ALzgYHORJB0e/gSyJMkwkCQZBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCQxsdterk3ySJK7O7WPJtnWHruG74CWZEmSf+qsu67T55QkdyUZTHJNu8UlSY5PsinJzvY8/zC8T0nSOCayZ3A9sLxbqKrXVtWyqloG3AJ8vLP6vuF1VXVxp34t8EZgaXsMj7ka2FxVS4HN7bUkaQYdNAyq6vPAvtHWtW/3rwFuHG+MJCcCx1bVlnZbzBuAc9vqFcC6tryuU5ckzZCpnjN4KfBwVe3s1E5K8pUkn0vy0lZbCOzutNndagALqmpvW34IWDDWxpKsSjKQZGBoaGiKU5ckDZtqGJzPD+4V7AWeXVUvBH4d+EiSYyc6WNtrqHHWr6mq/qrq7+vrm+ycJUkjHDXZjkmOAl4NnDJcq6r9wP62fEeS+4DnAnuARZ3ui1oN4OEkJ1bV3nY46ZHJzkmSNDlT2TP4j8DXqur7h3+S9CWZ15Z/mt6J4vvbYaDHk5zWzjNcANzWuq0HVrbllZ26JGmGTOTS0huBvwF+JsnuJBe1Vefx5BPHLwO2t0tN/wy4uKqGTz6/GfhTYBC4D/hkq18FvCLJTnoBc9Xk344kaTIOepioqs4fo37hKLVb6F1qOlr7AeAFo9S/AZxxsHlIkg4ffwJZkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJKY2J3O1iZ5JMndndoVSfYk2dYe53TWXZZkMMm9Sc7q1Je32mCS1Z36SUm+1OofTXL0dL5BSdLBTWTP4Hpg+Sj191bVsvbYAJDkZHq3w3x+6/PHSea1+yK/HzgbOBk4v7UF+IM21r8CHgUuGrkhSdLhddAwqKrPA/sO1q5ZAdxUVfur6uv07nd8ansMVtX9VfXPwE3AiiQB/gO9+yUDrAPOPbS3IEmaqqmcM7g0yfZ2GGl+qy0EHuy02d1qY9V/AviHqjowoj6qJKuSDCQZGBoamsLUJUldkw2Da4HnAMuAvcDV0zWh8VTVmqrqr6r+vr6+mdikJB0RjppMp6p6eHg5yQeA29vLPcDiTtNFrcYY9W8AxyU5qu0ddNtLkmbIpPYMkpzYefkqYPhKo/XAeUmOSXISsBT4MrAVWNquHDqa3knm9VVVwGeAX2r9VwK3TWZOkqTJO+ieQZIbgdOBE5LsBi4HTk+yDChgF/AmgKrakeRm4B7gAHBJVT3RxrkU2AjMA9ZW1Y62id8Gbkry+8BXgA9O15uTJE3MQcOgqs4fpTzmf9hVdSVw5Sj1DcCGUer307vaSJI0S/wJZEmSYSBJMgwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJYgJhkGRtkkeS3N2pvTvJ15JsT3JrkuNafUmSf0qyrT2u6/Q5JcldSQaTXJMkrX58kk1Jdrbn+YfhfUqSxjGRPYPrgeUjapuAF1TVzwL/B7iss+6+qlrWHhd36tcCb6R3X+SlnTFXA5uraimwub2WJM2gg4ZBVX0e2Dei9ldVdaC93AIsGm+MJCcCx1bVlqoq4Abg3LZ6BbCuLa/r1CVJM2Q6zhm8Afhk5/VJSb6S5HNJXtpqC4HdnTa7Ww1gQVXtbcsPAQvG2lCSVUkGkgwMDQ1Nw9QlSTDFMEjyO8AB4MOttBd4dlW9EPh14CNJjp3oeG2vocZZv6aq+quqv6+vbwozlyR1HTXZjkkuBP4TcEb7T5yq2g/sb8t3JLkPeC6whx88lLSo1QAeTnJiVe1th5MemeycJEmTM6k9gyTLgd8CfrGqvt2p9yWZ15Z/mt6J4vvbYaDHk5zWriK6ALitdVsPrGzLKzt1SdIMOeieQZIbgdOBE5LsBi6nd/XQMcCmdoXolnbl0MuAtyf5LvA94OKqGj75/GZ6VyY9nd45huHzDFcBNye5CHgAeM20vDNJ0oQdNAyq6vxRyh8co+0twC1jrBsAXjBK/RvAGQebhyTp8PEnkCVJhoEkyTCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kSEwyDJGuTPJLk7k7t+CSbkuxsz/NbPUmuSTKYZHuSF3X6rGztdyZZ2amfkuSu1ueadjc0SdIMmeiewfXA8hG11cDmqloKbG6vAc6md7vLpcAq4FrohQe9u6S9GDgVuHw4QFqbN3b6jdyWJOkwmlAYVNXngX0jyiuAdW15HXBup35D9WwBjms3uj8L2FRV+6rqUWATsLytO7aqtlRVATd0xpIkzYCpnDNY0G50D/AQsKAtLwQe7LTb3Wrj1XePUpckzZBpOYHcvtHXdIw1niSrkgwkGRgaGjrcm5OkI8ZUwuDhdoiH9vxIq+8BFnfaLWq18eqLRqk/SVWtqar+qurv6+ubwtQlSV1TCYP1wPAVQSuB2zr1C9pVRacBj7XDSRuBM5PMbyeOzwQ2tnWPJzmtXUV0QWcsSdIMOGoijZLcCJwOnJBkN72rgq4Cbk5yEfAA8JrWfANwDjAIfBt4PUBV7UvyDmBra/f2qho+Kf1melcsPR34ZHtIkmbIhMKgqs4fY9UZo7Qt4JIxxlkLrB2lPgC8YCJzkSRNP38CWZJkGEiSDANJEoaBJAnDQJLEBK8mkjSzlqz+xGxPQXPUrqteeVjGdc9AkmQYSJIMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkphAGSX4mybbO4/Ekb0lyRZI9nfo5nT6XJRlMcm+Sszr15a02mGT1VN+UJOnQTPp3E1XVvcAygCTz6N3E/lZ6t7l8b1W9p9s+ycnAecDzgZ8CPpXkuW31+4FXALuBrUnWV9U9k52bJOnQTNcvqjsDuK+qHujd035UK4Cbqmo/8PUkg8Cpbd1gVd0PkOSm1tYwkKQZMl3nDM4Dbuy8vjTJ9iRrk8xvtYXAg502u1ttrPqTJFmVZCDJwNDQ0DRNXZI05TBIcjTwi8DHWula4Dn0DiHtBa6e6jaGVdWaquqvqv6+vr7pGlaSjnjTcZjobODOqnoYYPgZIMkHgNvbyz3A4k6/Ra3GOHVJ0gyYjsNE59M5RJTkxM66VwF3t+X1wHlJjklyErAU+DKwFVia5KS2l3FeaytJmiFT2jNI8mP0rgJ6U6f8riTLgAJ2Da+rqh1JbqZ3YvgAcElVPdHGuRTYCMwD1lbVjqnMS5J0aKYUBlX1j8BPjKi9bpz2VwJXjlLfAGyYylwkSZPnTyBLkgwDSZJhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRLTEAZJdiW5K8m2JAOtdnySTUl2tuf5rZ4k1yQZTLI9yYs646xs7XcmWTnVeUmSJm669gxeXlXLqqq/vV4NbK6qpcDm9hrgbHr3Pl4KrAKuhV54AJcDLwZOBS4fDhBJ0uF3uA4TrQDWteV1wLmd+g3VswU4LsmJwFnApqraV1WPApuA5YdpbpKkEaYjDAr4qyR3JFnVaguqam9bfghY0JYXAg92+u5utbHqPyDJqiQDSQaGhoamYeqSJICjpmGMn6+qPUmeBWxK8rXuyqqqJDUN26Gq1gBrAPr7+6dlTEnSNOwZVNWe9vwIcCu9Y/4Pt8M/tOdHWvM9wOJO90WtNlZdkjQDphQGSX4syY8PLwNnAncD64HhK4JWAre15fXABe2qotOAx9rhpI3AmUnmtxPHZ7aaJGkGTPUw0QLg1iTDY32kqv4yyVbg5iQXAQ8Ar2ntNwDnAIPAt4HXA1TVviTvALa2dm+vqn1TnJskaYKmFAZVdT/wc6PUvwGcMUq9gEvGGGstsHYq85EkTY4/gSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSUwhDJIsTvKZJPck2ZHkV1v9iiR7kmxrj3M6fS5LMpjk3iRnderLW20wyeqpvSVJ0qGayp3ODgC/UVV3tvsg35FkU1v33qp6T7dxkpOB84DnAz8FfCrJc9vq9wOvAHYDW5Osr6p7pjA3SdIhmHQYtBvZ723L30zyVWDhOF1WADdV1X7g60kGgVPbusF2C02S3NTaGgaSNEOm5ZxBkiXAC4EvtdKlSbYnWZtkfqstBB7sdNvdamPVR9vOqiQDSQaGhoamY+qSJKYhDJI8A7gFeEtVPQ5cCzwHWEZvz+HqqW5jWFWtqar+qurv6+ubrmEl6Yg3lXMGJHkavSD4cFV9HKCqHu6s/wBwe3u5B1jc6b6o1RinLkmaAVO5mijAB4GvVtUfduondpq9Cri7La8HzktyTJKTgKXAl4GtwNIkJyU5mt5J5vWTnZck6dBNZc/gJcDrgLuSbGu1twHnJ1kGFLALeBNAVe1IcjO9E8MHgEuq6gmAJJcCG4F5wNqq2jGFeUmSDtFUrib6IpBRVm0Yp8+VwJWj1DeM10+SdHj5E8iSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAksQcCoMky5Pcm2QwyerZno8kHUnmRBgkmQe8HzgbOJnerTNPnt1ZSdKRY06EAXAqMFhV91fVPwM3AStmeU6SdMSY9D2Qp9lC4MHO693Ai0c2SrIKWNVefivJvTMwtyPBCcDfz/Yk5oL8wWzPQGPwM9pMw2f0X45WnCthMCFVtQZYM9vz+GGTZKCq+md7HtJY/IwefnPlMNEeYHHn9aJWkyTNgLkSBluBpUlOSnI0cB6wfpbnJElHjDlxmKiqDiS5FNgIzAPWVtWOWZ7WkcRDb5rr/IweZqmq2Z6DJGmWzZXDRJKkWWQYSJIMg7koyauSbBvx+F6Ssw/Dtn4yyU1J7ktyR5INSZ47zds4Pcm/n84xNf2SVJKrO6/fmuSKaRp744jP898l+dJ0jD3Kti5IcneSu5J8JclbD8M23jbdY842w2AOqqpbq2rZ8AP4Y+AL9E6wH1R6Dvp3myTArcBnq+o5VXUKcBmwYPKzH9XpgGEw9+0HXp3khOkeuKrO6nyeXwI8DvzuRPsnmdDFLu0L01uAM6vq3wCnAY8d8oQPzjDQzGrf0v8n8Lqq+l6r/WaSrUm2J/m9VlvSftHfDcDdwOIk7+58Q3rtKMO/HPhuVV03XKiqv62qL7RAeVL/9i3/9s78/ijJhW15V5LfS3Jn6/O8JEuAi4Ffa98IX3o4/pw0LQ7Qu2rn10auaJ+vT7fP3OYkz27165Nck+Svk9yf5JcmsJ3/BWyoqk1tjOck+cu2Z/qFJM/rjH1d24N4V5JlSba0OdyaZP4oY18GvLWq/g6gqvZX1QfaeKP2T/LZJP1t+YQku9ryhUk+3ua2M8m7Wv0q4Ont8/zhCf/pznVV5WOOPoCnAQPAazu1M+n9gw29ML8deBmwBPgecFpr95+BTfQu1V0A/F/gxBHj/w/gvWNse9T+9L7l395p90fAhW15F/ArbfnNwJ+25Svo/QOd9T9TH+N+3r4FHNv+Hp8JvBW4oq37C2BlW34D8Odt+XrgY+2zeDK93zE23jZeDWwDjunUNgNL2/KLgU93xr4dmNdebwd+oS2/HXjfKOPvA545xrZH7Q98FuhvyycAu9ryhcD97c/iR4EHgMXDf1az/fc13Q/3DOa2dwA7quqjndqZ7fEV4E7gecDStu6BqtrSln8euLGqnqiqh4HPAf/2ELY92f4fb8930AsoPYVU1ePADfS+KHT9O+Ajbfl/0/t8DPvzqvpeVd3DOIcYkyykt1fwX6tqf6s9g94hxI8l2Qb8Cb0vHcM+VlVPJHkmcFxVfa7V19H7EjQhU+i/uaoeq6rvAPcwxu/1+WEwJ37oTE+W5HR6385fNHIV8M6q+pMR7ZcA/3iIm9kBTGS3vusAP3h48UdHrN/fnp/Az9dT1fvofdH40ATb7+8sByDJlcArAapqWTs/tQ64qoXGsB8B/qF65xJGM5nP9CnApw+hT/czPdbnGX7IP9PuGcxB7Vjmh4ALquqbI1ZvBN7QvlGRZGGSZ40yzBeA1yaZl6SP3regL49o82ngmPR+G+zwtn+2Hdcfq/8DwMlJjklyHHDGBN7SN4Efn0A7zQFVtQ+4GbioU/5rer8mBuCX6X0+xhvjd+r/nzCG3iGn71TV+0e0exz4epL/At+/+OHnRhnvMeDRzjmn19HbWx3pncC7k/xkG+/oJP/9IP130QsQmPiXo+8medoE2z4l/NCm3FPcxcCzgGt7X6i+751V9dEk/xr4m7buW8B/o/etpetWerv2fwsU8FtV9VC3QVVVklcB70vy28B36P3DeAvwxbH6J7mZ3knqr9M7XHUwfwH8WZIV9M4pjPsfieaEq4FLO69/BfhQkt8EhoDXH+J4vw/sboeChj1aVS+nFy7XJvldeufJbqL3uRtpJXBdkn9B71j+k+ZQVRuSLAA+1fZGClh7kP7vAW5uX4o+McH3swbYnuTOqvrlCfaZ0/x1FJIkDxNJkgwDSRKGgSQJw0CShGEgScIwkCRhGEiSgP8HDExfVKo3d8kAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     }
    }
   ],
   "source": [
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt\n",
    "zero_count = claim_data[(claim_data[\"Claim_Amount\"] == 0)].count()[0]\n",
    "non_zero_count = claim_data[claim_data[\"Claim_Amount\" ] > 0].count()[0]\n",
    "print(zero_count)\n",
    "print(non_zero_count)\n",
    "plt.bar([\"Zero Count\", \"Non-Zero Count\"], [zero_count, non_zero_count])"
   ]
  },
  {
   "source": [
    "There is 11919 more zero claim pieces of data within the dataset. This is roughly double the amount of non zero claims and this will create a bias in the dataset. This would make there be less chance of correctly predicting when there should be a claim value. I am going to implement undersampiling on the data to try and address this balance down to a reasonable level. I am going to reduce the gap between the two down to an 80% ratio between the two classes. This will leave the current dataset to around 16000 samples still which is over half and still more than enough to work with for effective results. Below I will create a function to call my sampaling technique on, but I will not use it on the data set until I seperate out the data in the next section for this reason."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "#Take in features and labels, return under sampled labels\n",
    "def sampler(features):\n",
    "    #First of all make the continous data binary\n",
    "    undersample = RandomUnderSampler(sampling_strategy=0.8)\n",
    "    features, y = undersample.fit_resample(features,features[\"Binary\"])\n",
    "    #Shuffle the features so they are mixed\n",
    "    features = features.sample(frac=1).reset_index(drop=True)\n",
    "    return features"
   ]
  },
  {
   "source": [
    "Question 1 - Data Preprocessing - Additional Steps - Feature Selection\n",
    "\n",
    "In this step I will investigate which features to use in the model and which ones I beleive can be dropped. I have already removed some variables such as Blind_Model/Sub_Model due to the variation in the categories. Using a spearman correlation between variables I will generate my own correlation matrix."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[('Cat3', -0.04790492131918909),\n",
       " ('Cat6', -0.029224793349666923),\n",
       " ('Var6', -0.028089961538438533),\n",
       " ('Var2', -0.02581927135385686),\n",
       " ('Var3', -0.025584777184900275),\n",
       " ('Cat5', -0.025571210789607796),\n",
       " ('OrdCat', -0.02402822908539657),\n",
       " ('Var7', -0.023721559865740045),\n",
       " ('Var1', -0.02231187838834493),\n",
       " ('NVCat', -0.022172909145313884),\n",
       " ('Var4', -0.02164389398293612),\n",
       " ('Cat1', -0.01986258472283148),\n",
       " ('Cat8', -0.01788243275623102),\n",
       " ('Var8', -0.015325498609193241),\n",
       " ('Var5', -0.011650895649837257),\n",
       " ('Cat2', -0.008279476102650968),\n",
       " ('Calendar_Year', -0.0070099969964738645),\n",
       " ('Household_ID', -0.006542315689061522),\n",
       " ('Cat4', -0.0031628205153692544),\n",
       " ('Model_Year', -0.0025113220633293247),\n",
       " ('Blind_Make', -0.00141182169136917),\n",
       " ('Cat11', -0.0002897867019988965),\n",
       " ('Cat10', 0.00024213641111735145),\n",
       " ('Vehicle', 0.005314862327437765),\n",
       " ('Cat12', 0.005502589727205273),\n",
       " ('NVVar4', 0.01820662879434113),\n",
       " ('NVVar1', 0.025784925890525546),\n",
       " ('Cat7', 0.029546734554520282),\n",
       " ('Cat9', 0.0316297935525209),\n",
       " ('NVVar3', 0.07672757891375388),\n",
       " ('NVVar2', 0.08141591903458949)]"
      ]
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "from scipy import stats\n",
    "from operator import itemgetter\n",
    "corr_matrix = []\n",
    "for feature in attributes_cat:\n",
    "    corr_matrix.append((feature, stats.spearmanr(claim_data[feature], claim_data[\"Claim_Amount\"])[0]))\n",
    "for feature in attributes_num:\n",
    "    corr_matrix.append((feature, stats.spearmanr(claim_data[feature], claim_data[\"Claim_Amount\"])[0]))\n",
    "for feature in attributes_other:\n",
    "    corr_matrix.append((feature, stats.spearmanr(claim_data[feature], claim_data[\"Claim_Amount\"])[0]))\n",
    "sorted(corr_matrix,key=itemgetter(1))"
   ]
  },
  {
   "source": [
    "The correlation here is showing there to be very little correlation between all the features and predicting the claim amount. The closer to 0 the more independent the variable and the less it should help the model. I have decided to cut anything between the values of +/- 0.01. This means the feature list I will use is as follows. I am left with 21 features and the target variable of Claim_Amount."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "      Cat1 Cat3 Cat5 Cat6 Cat7 Cat8 Cat9  OrdCat      Var1      Var2  ...  \\\n",
       "0        B    A    A    C    C    B    B     2.0 -1.172101 -1.248430  ...   \n",
       "1        F    A    A    C    D    A    B     2.0 -0.641792 -0.643240  ...   \n",
       "2        B    A    A    C    C    B    B     2.0 -0.770352 -1.507798  ...   \n",
       "3        B    A    A    C    C    A    B     2.0 -0.698037  1.448988  ...   \n",
       "4        I    B    A    D    A    B    B     5.0  0.322405  1.691064  ...   \n",
       "...    ...  ...  ...  ...  ...  ...  ...     ...       ...       ...  ...   \n",
       "29830    D    A    A    C    A    A    B     5.0  1.519617  0.826507  ...   \n",
       "29831    F    A    A    C    A    B    B     2.0 -0.633757 -0.643240  ...   \n",
       "29832    B    E    A    B    C    A    B     4.0 -0.481093 -0.816152  ...   \n",
       "29833    B    E    A    B    C    A    B     4.0 -0.481093 -0.989063  ...   \n",
       "29834    F    A    A    B    D    A    B     4.0 -0.665897 -0.418456  ...   \n",
       "\n",
       "           Var5      Var6      Var7      Var8  NVCat   NVVar1    NVVar2  \\\n",
       "0     -1.054747 -1.164837 -0.812472 -0.581756      N -0.23153 -0.266117   \n",
       "1     -1.149901 -1.299354 -1.193388 -1.323992      N -0.23153  2.783616   \n",
       "2     -0.775231 -1.467792 -0.947636 -0.914860      F -0.23153 -0.266117   \n",
       "3     -0.751442 -0.378792 -0.689596  0.730640      M -0.23153 -0.266117   \n",
       "4      0.461775  1.451805  1.411586  0.977104      B -0.23153 -0.266117   \n",
       "...         ...       ...       ...       ...    ...      ...       ...   \n",
       "29830  1.425212  0.907890  0.711192  0.194939      M -0.23153 -0.266117   \n",
       "29831 -0.608711 -1.134425 -1.193388 -0.844848      L -0.23153 -0.266117   \n",
       "29832 -0.162675 -0.597528 -1.058225 -0.728890      N -0.23153 -0.266117   \n",
       "29833  0.051422 -0.578812 -1.058225 -0.656034      M -0.23153 -0.266117   \n",
       "29834 -0.055627 -0.829130 -1.193388 -0.906108      M -0.23153 -0.266117   \n",
       "\n",
       "         NVVar3    NVVar4  Claim_Amount  \n",
       "0      2.715490 -0.251419    104.597500  \n",
       "1     -0.272337 -0.251419      0.000000  \n",
       "2     -0.272337 -0.251419      0.000000  \n",
       "3     -0.272337 -0.251419      0.000000  \n",
       "4     -0.272337 -0.251419      0.615937  \n",
       "...         ...       ...           ...  \n",
       "29830 -0.272337 -0.251419      0.000000  \n",
       "29831  4.209404 -0.251419    135.602700  \n",
       "29832  2.715490 -0.251419     42.755250  \n",
       "29833 -0.272337 -0.251419      0.000000  \n",
       "29834 -0.272337 -0.251419      0.000000  \n",
       "\n",
       "[29835 rows x 22 columns]"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Cat1</th>\n      <th>Cat3</th>\n      <th>Cat5</th>\n      <th>Cat6</th>\n      <th>Cat7</th>\n      <th>Cat8</th>\n      <th>Cat9</th>\n      <th>OrdCat</th>\n      <th>Var1</th>\n      <th>Var2</th>\n      <th>...</th>\n      <th>Var5</th>\n      <th>Var6</th>\n      <th>Var7</th>\n      <th>Var8</th>\n      <th>NVCat</th>\n      <th>NVVar1</th>\n      <th>NVVar2</th>\n      <th>NVVar3</th>\n      <th>NVVar4</th>\n      <th>Claim_Amount</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>B</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>C</td>\n      <td>B</td>\n      <td>B</td>\n      <td>2.0</td>\n      <td>-1.172101</td>\n      <td>-1.248430</td>\n      <td>...</td>\n      <td>-1.054747</td>\n      <td>-1.164837</td>\n      <td>-0.812472</td>\n      <td>-0.581756</td>\n      <td>N</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>2.715490</td>\n      <td>-0.251419</td>\n      <td>104.597500</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>F</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>D</td>\n      <td>A</td>\n      <td>B</td>\n      <td>2.0</td>\n      <td>-0.641792</td>\n      <td>-0.643240</td>\n      <td>...</td>\n      <td>-1.149901</td>\n      <td>-1.299354</td>\n      <td>-1.193388</td>\n      <td>-1.323992</td>\n      <td>N</td>\n      <td>-0.23153</td>\n      <td>2.783616</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>B</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>C</td>\n      <td>B</td>\n      <td>B</td>\n      <td>2.0</td>\n      <td>-0.770352</td>\n      <td>-1.507798</td>\n      <td>...</td>\n      <td>-0.775231</td>\n      <td>-1.467792</td>\n      <td>-0.947636</td>\n      <td>-0.914860</td>\n      <td>F</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>B</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>C</td>\n      <td>A</td>\n      <td>B</td>\n      <td>2.0</td>\n      <td>-0.698037</td>\n      <td>1.448988</td>\n      <td>...</td>\n      <td>-0.751442</td>\n      <td>-0.378792</td>\n      <td>-0.689596</td>\n      <td>0.730640</td>\n      <td>M</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>I</td>\n      <td>B</td>\n      <td>A</td>\n      <td>D</td>\n      <td>A</td>\n      <td>B</td>\n      <td>B</td>\n      <td>5.0</td>\n      <td>0.322405</td>\n      <td>1.691064</td>\n      <td>...</td>\n      <td>0.461775</td>\n      <td>1.451805</td>\n      <td>1.411586</td>\n      <td>0.977104</td>\n      <td>B</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.615937</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>29830</th>\n      <td>D</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>A</td>\n      <td>A</td>\n      <td>B</td>\n      <td>5.0</td>\n      <td>1.519617</td>\n      <td>0.826507</td>\n      <td>...</td>\n      <td>1.425212</td>\n      <td>0.907890</td>\n      <td>0.711192</td>\n      <td>0.194939</td>\n      <td>M</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>29831</th>\n      <td>F</td>\n      <td>A</td>\n      <td>A</td>\n      <td>C</td>\n      <td>A</td>\n      <td>B</td>\n      <td>B</td>\n      <td>2.0</td>\n      <td>-0.633757</td>\n      <td>-0.643240</td>\n      <td>...</td>\n      <td>-0.608711</td>\n      <td>-1.134425</td>\n      <td>-1.193388</td>\n      <td>-0.844848</td>\n      <td>L</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>4.209404</td>\n      <td>-0.251419</td>\n      <td>135.602700</td>\n    </tr>\n    <tr>\n      <th>29832</th>\n      <td>B</td>\n      <td>E</td>\n      <td>A</td>\n      <td>B</td>\n      <td>C</td>\n      <td>A</td>\n      <td>B</td>\n      <td>4.0</td>\n      <td>-0.481093</td>\n      <td>-0.816152</td>\n      <td>...</td>\n      <td>-0.162675</td>\n      <td>-0.597528</td>\n      <td>-1.058225</td>\n      <td>-0.728890</td>\n      <td>N</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>2.715490</td>\n      <td>-0.251419</td>\n      <td>42.755250</td>\n    </tr>\n    <tr>\n      <th>29833</th>\n      <td>B</td>\n      <td>E</td>\n      <td>A</td>\n      <td>B</td>\n      <td>C</td>\n      <td>A</td>\n      <td>B</td>\n      <td>4.0</td>\n      <td>-0.481093</td>\n      <td>-0.989063</td>\n      <td>...</td>\n      <td>0.051422</td>\n      <td>-0.578812</td>\n      <td>-1.058225</td>\n      <td>-0.656034</td>\n      <td>M</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n    <tr>\n      <th>29834</th>\n      <td>F</td>\n      <td>A</td>\n      <td>A</td>\n      <td>B</td>\n      <td>D</td>\n      <td>A</td>\n      <td>B</td>\n      <td>4.0</td>\n      <td>-0.665897</td>\n      <td>-0.418456</td>\n      <td>...</td>\n      <td>-0.055627</td>\n      <td>-0.829130</td>\n      <td>-1.193388</td>\n      <td>-0.906108</td>\n      <td>M</td>\n      <td>-0.23153</td>\n      <td>-0.266117</td>\n      <td>-0.272337</td>\n      <td>-0.251419</td>\n      <td>0.000000</td>\n    </tr>\n  </tbody>\n</table>\n<p>29835 rows × 22 columns</p>\n</div>"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "claim_data_selected = claim_data.drop([\"Row_ID\",\"Cat12\", \"Vehicle\", \"Cat10\", \"Cat11\", \"Blind_Make\", \"Model_Year\",\"Cat4\",\"Household_ID\", \"Calendar_Year\", \"Cat2\"], axis=1)\n",
    "claim_data_selected"
   ]
  },
  {
   "source": [
    "Question 2 - Peformance Using a single model - Linear Regression\n",
    "\n",
    "All worth [2 marks].\n",
    "\n",
    "For each model, use grid search with at least three options for each parameter and clearly report the performance measure over a validation set.\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "First I am going to create a lot of variables that will be needed for the next few questions. I have chose to do that all here to keep it as central as possible. Hopefully my comments will make it clear what each thing is doing. I will then use the classifier listed in each question and then compare at the end. The MSE should be stated once the model is done."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "8910\n7128\nThe MSE for Linear Regression is 49089.68171262561\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 432x288 with 1 Axes>",
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n<svg height=\"248.518125pt\" version=\"1.1\" viewBox=\"0 0 381.65 248.518125\" width=\"381.65pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n <metadata>\r\n  <rdf:RDF xmlns:cc=\"http://creativecommons.org/ns#\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\r\n   <cc:Work>\r\n    <dc:type rdf:resource=\"http://purl.org/dc/dcmitype/StillImage\"/>\r\n    <dc:date>2020-10-29T18:11:07.959391</dc:date>\r\n    <dc:format>image/svg+xml</dc:format>\r\n    <dc:creator>\r\n     <cc:Agent>\r\n      <dc:title>Matplotlib v3.3.2, https://matplotlib.org/</dc:title>\r\n     </cc:Agent>\r\n    </dc:creator>\r\n   </cc:Work>\r\n  </rdf:RDF>\r\n </metadata>\r\n <defs>\r\n  <style type=\"text/css\">*{stroke-linecap:butt;stroke-linejoin:round;}</style>\r\n </defs>\r\n <g id=\"figure_1\">\r\n  <g id=\"patch_1\">\r\n   <path d=\"M -0 248.518125 \r\nL 381.65 248.518125 \r\nL 381.65 0 \r\nL -0 0 \r\nz\r\n\" style=\"fill:none;\"/>\r\n  </g>\r\n  <g id=\"axes_1\">\r\n   <g id=\"patch_2\">\r\n    <path d=\"M 39.65 224.64 \r\nL 374.45 224.64 \r\nL 374.45 7.2 \r\nL 39.65 7.2 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n   </g>\r\n   <g id=\"patch_3\">\r\n    <path clip-path=\"url(#p9c207e56d7)\" d=\"M 54.868182 224.64 \r\nL 190.140909 224.64 \r\nL 190.140909 17.554286 \r\nL 54.868182 17.554286 \r\nz\r\n\" style=\"fill:#1f77b4;\"/>\r\n   </g>\r\n   <g id=\"patch_4\">\r\n    <path clip-path=\"url(#p9c207e56d7)\" d=\"M 223.959091 224.64 \r\nL 359.231818 224.64 \r\nL 359.231818 58.971429 \r\nL 223.959091 58.971429 \r\nz\r\n\" style=\"fill:#1f77b4;\"/>\r\n   </g>\r\n   <g id=\"matplotlib.axis_1\">\r\n    <g id=\"xtick_1\">\r\n     <g id=\"line2d_1\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL 0 3.5 \r\n\" id=\"m32f9e6033a\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"122.504545\" xlink:href=\"#m32f9e6033a\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_1\">\r\n      <!-- Zero Count -->\r\n      <g transform=\"translate(94.563139 239.238437)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 5.609375 72.90625 \r\nL 62.890625 72.90625 \r\nL 62.890625 65.375 \r\nL 16.796875 8.296875 \r\nL 64.015625 8.296875 \r\nL 64.015625 0 \r\nL 4.5 0 \r\nL 4.5 7.515625 \r\nL 50.59375 64.59375 \r\nL 5.609375 64.59375 \r\nz\r\n\" id=\"DejaVuSans-90\"/>\r\n        <path d=\"M 56.203125 29.59375 \r\nL 56.203125 25.203125 \r\nL 14.890625 25.203125 \r\nQ 15.484375 15.921875 20.484375 11.0625 \r\nQ 25.484375 6.203125 34.421875 6.203125 \r\nQ 39.59375 6.203125 44.453125 7.46875 \r\nQ 49.3125 8.734375 54.109375 11.28125 \r\nL 54.109375 2.78125 \r\nQ 49.265625 0.734375 44.1875 -0.34375 \r\nQ 39.109375 -1.421875 33.890625 -1.421875 \r\nQ 20.796875 -1.421875 13.15625 6.1875 \r\nQ 5.515625 13.8125 5.515625 26.8125 \r\nQ 5.515625 40.234375 12.765625 48.109375 \r\nQ 20.015625 56 32.328125 56 \r\nQ 43.359375 56 49.78125 48.890625 \r\nQ 56.203125 41.796875 56.203125 29.59375 \r\nz\r\nM 47.21875 32.234375 \r\nQ 47.125 39.59375 43.09375 43.984375 \r\nQ 39.0625 48.390625 32.421875 48.390625 \r\nQ 24.90625 48.390625 20.390625 44.140625 \r\nQ 15.875 39.890625 15.1875 32.171875 \r\nz\r\n\" id=\"DejaVuSans-101\"/>\r\n        <path d=\"M 41.109375 46.296875 \r\nQ 39.59375 47.171875 37.8125 47.578125 \r\nQ 36.03125 48 33.890625 48 \r\nQ 26.265625 48 22.1875 43.046875 \r\nQ 18.109375 38.09375 18.109375 28.8125 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 20.953125 51.171875 25.484375 53.578125 \r\nQ 30.03125 56 36.53125 56 \r\nQ 37.453125 56 38.578125 55.875 \r\nQ 39.703125 55.765625 41.0625 55.515625 \r\nz\r\n\" id=\"DejaVuSans-114\"/>\r\n        <path d=\"M 30.609375 48.390625 \r\nQ 23.390625 48.390625 19.1875 42.75 \r\nQ 14.984375 37.109375 14.984375 27.296875 \r\nQ 14.984375 17.484375 19.15625 11.84375 \r\nQ 23.34375 6.203125 30.609375 6.203125 \r\nQ 37.796875 6.203125 41.984375 11.859375 \r\nQ 46.1875 17.53125 46.1875 27.296875 \r\nQ 46.1875 37.015625 41.984375 42.703125 \r\nQ 37.796875 48.390625 30.609375 48.390625 \r\nz\r\nM 30.609375 56 \r\nQ 42.328125 56 49.015625 48.375 \r\nQ 55.71875 40.765625 55.71875 27.296875 \r\nQ 55.71875 13.875 49.015625 6.21875 \r\nQ 42.328125 -1.421875 30.609375 -1.421875 \r\nQ 18.84375 -1.421875 12.171875 6.21875 \r\nQ 5.515625 13.875 5.515625 27.296875 \r\nQ 5.515625 40.765625 12.171875 48.375 \r\nQ 18.84375 56 30.609375 56 \r\nz\r\n\" id=\"DejaVuSans-111\"/>\r\n        <path id=\"DejaVuSans-32\"/>\r\n        <path d=\"M 64.40625 67.28125 \r\nL 64.40625 56.890625 \r\nQ 59.421875 61.53125 53.78125 63.8125 \r\nQ 48.140625 66.109375 41.796875 66.109375 \r\nQ 29.296875 66.109375 22.65625 58.46875 \r\nQ 16.015625 50.828125 16.015625 36.375 \r\nQ 16.015625 21.96875 22.65625 14.328125 \r\nQ 29.296875 6.6875 41.796875 6.6875 \r\nQ 48.140625 6.6875 53.78125 8.984375 \r\nQ 59.421875 11.28125 64.40625 15.921875 \r\nL 64.40625 5.609375 \r\nQ 59.234375 2.09375 53.4375 0.328125 \r\nQ 47.65625 -1.421875 41.21875 -1.421875 \r\nQ 24.65625 -1.421875 15.125 8.703125 \r\nQ 5.609375 18.84375 5.609375 36.375 \r\nQ 5.609375 53.953125 15.125 64.078125 \r\nQ 24.65625 74.21875 41.21875 74.21875 \r\nQ 47.75 74.21875 53.53125 72.484375 \r\nQ 59.328125 70.75 64.40625 67.28125 \r\nz\r\n\" id=\"DejaVuSans-67\"/>\r\n        <path d=\"M 8.5 21.578125 \r\nL 8.5 54.6875 \r\nL 17.484375 54.6875 \r\nL 17.484375 21.921875 \r\nQ 17.484375 14.15625 20.5 10.265625 \r\nQ 23.53125 6.390625 29.59375 6.390625 \r\nQ 36.859375 6.390625 41.078125 11.03125 \r\nQ 45.3125 15.671875 45.3125 23.6875 \r\nL 45.3125 54.6875 \r\nL 54.296875 54.6875 \r\nL 54.296875 0 \r\nL 45.3125 0 \r\nL 45.3125 8.40625 \r\nQ 42.046875 3.421875 37.71875 1 \r\nQ 33.40625 -1.421875 27.6875 -1.421875 \r\nQ 18.265625 -1.421875 13.375 4.4375 \r\nQ 8.5 10.296875 8.5 21.578125 \r\nz\r\nM 31.109375 56 \r\nz\r\n\" id=\"DejaVuSans-117\"/>\r\n        <path d=\"M 54.890625 33.015625 \r\nL 54.890625 0 \r\nL 45.90625 0 \r\nL 45.90625 32.71875 \r\nQ 45.90625 40.484375 42.875 44.328125 \r\nQ 39.84375 48.1875 33.796875 48.1875 \r\nQ 26.515625 48.1875 22.3125 43.546875 \r\nQ 18.109375 38.921875 18.109375 30.90625 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 21.34375 51.125 25.703125 53.5625 \r\nQ 30.078125 56 35.796875 56 \r\nQ 45.21875 56 50.046875 50.171875 \r\nQ 54.890625 44.34375 54.890625 33.015625 \r\nz\r\n\" id=\"DejaVuSans-110\"/>\r\n        <path d=\"M 18.3125 70.21875 \r\nL 18.3125 54.6875 \r\nL 36.8125 54.6875 \r\nL 36.8125 47.703125 \r\nL 18.3125 47.703125 \r\nL 18.3125 18.015625 \r\nQ 18.3125 11.328125 20.140625 9.421875 \r\nQ 21.96875 7.515625 27.59375 7.515625 \r\nL 36.8125 7.515625 \r\nL 36.8125 0 \r\nL 27.59375 0 \r\nQ 17.1875 0 13.234375 3.875 \r\nQ 9.28125 7.765625 9.28125 18.015625 \r\nL 9.28125 47.703125 \r\nL 2.6875 47.703125 \r\nL 2.6875 54.6875 \r\nL 9.28125 54.6875 \r\nL 9.28125 70.21875 \r\nz\r\n\" id=\"DejaVuSans-116\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-90\"/>\r\n       <use x=\"68.505859\" xlink:href=\"#DejaVuSans-101\"/>\r\n       <use x=\"130.029297\" xlink:href=\"#DejaVuSans-114\"/>\r\n       <use x=\"168.892578\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"230.074219\" xlink:href=\"#DejaVuSans-32\"/>\r\n       <use x=\"261.861328\" xlink:href=\"#DejaVuSans-67\"/>\r\n       <use x=\"331.685547\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"392.867188\" xlink:href=\"#DejaVuSans-117\"/>\r\n       <use x=\"456.246094\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"519.625\" xlink:href=\"#DejaVuSans-116\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_2\">\r\n     <g id=\"line2d_2\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"291.595455\" xlink:href=\"#m32f9e6033a\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_2\">\r\n      <!-- Non-Zero Count -->\r\n      <g transform=\"translate(251.881392 239.238437)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 9.8125 72.90625 \r\nL 23.09375 72.90625 \r\nL 55.421875 11.921875 \r\nL 55.421875 72.90625 \r\nL 64.984375 72.90625 \r\nL 64.984375 0 \r\nL 51.703125 0 \r\nL 19.390625 60.984375 \r\nL 19.390625 0 \r\nL 9.8125 0 \r\nz\r\n\" id=\"DejaVuSans-78\"/>\r\n        <path d=\"M 4.890625 31.390625 \r\nL 31.203125 31.390625 \r\nL 31.203125 23.390625 \r\nL 4.890625 23.390625 \r\nz\r\n\" id=\"DejaVuSans-45\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-78\"/>\r\n       <use x=\"74.804688\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"135.986328\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"199.365234\" xlink:href=\"#DejaVuSans-45\"/>\r\n       <use x=\"235.449219\" xlink:href=\"#DejaVuSans-90\"/>\r\n       <use x=\"303.955078\" xlink:href=\"#DejaVuSans-101\"/>\r\n       <use x=\"365.478516\" xlink:href=\"#DejaVuSans-114\"/>\r\n       <use x=\"404.341797\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"465.523438\" xlink:href=\"#DejaVuSans-32\"/>\r\n       <use x=\"497.310547\" xlink:href=\"#DejaVuSans-67\"/>\r\n       <use x=\"567.134766\" xlink:href=\"#DejaVuSans-111\"/>\r\n       <use x=\"628.316406\" xlink:href=\"#DejaVuSans-117\"/>\r\n       <use x=\"691.695312\" xlink:href=\"#DejaVuSans-110\"/>\r\n       <use x=\"755.074219\" xlink:href=\"#DejaVuSans-116\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"matplotlib.axis_2\">\r\n    <g id=\"ytick_1\">\r\n     <g id=\"line2d_3\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL -3.5 0 \r\n\" id=\"m80e23da729\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"39.65\" xlink:href=\"#m80e23da729\" y=\"224.64\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_3\">\r\n      <!-- 0 -->\r\n      <g transform=\"translate(26.2875 228.439219)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 31.78125 66.40625 \r\nQ 24.171875 66.40625 20.328125 58.90625 \r\nQ 16.5 51.421875 16.5 36.375 \r\nQ 16.5 21.390625 20.328125 13.890625 \r\nQ 24.171875 6.390625 31.78125 6.390625 \r\nQ 39.453125 6.390625 43.28125 13.890625 \r\nQ 47.125 21.390625 47.125 36.375 \r\nQ 47.125 51.421875 43.28125 58.90625 \r\nQ 39.453125 66.40625 31.78125 66.40625 \r\nz\r\nM 31.78125 74.21875 \r\nQ 44.046875 74.21875 50.515625 64.515625 \r\nQ 56.984375 54.828125 56.984375 36.375 \r\nQ 56.984375 17.96875 50.515625 8.265625 \r\nQ 44.046875 -1.421875 31.78125 -1.421875 \r\nQ 19.53125 -1.421875 13.0625 8.265625 \r\nQ 6.59375 17.96875 6.59375 36.375 \r\nQ 6.59375 54.828125 13.0625 64.515625 \r\nQ 19.53125 74.21875 31.78125 74.21875 \r\nz\r\n\" id=\"DejaVuSans-48\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_2\">\r\n     <g id=\"line2d_4\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"39.65\" xlink:href=\"#m80e23da729\" y=\"178.156114\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_4\">\r\n      <!-- 2000 -->\r\n      <g transform=\"translate(7.2 181.955332)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 19.1875 8.296875 \r\nL 53.609375 8.296875 \r\nL 53.609375 0 \r\nL 7.328125 0 \r\nL 7.328125 8.296875 \r\nQ 12.9375 14.109375 22.625 23.890625 \r\nQ 32.328125 33.6875 34.8125 36.53125 \r\nQ 39.546875 41.84375 41.421875 45.53125 \r\nQ 43.3125 49.21875 43.3125 52.78125 \r\nQ 43.3125 58.59375 39.234375 62.25 \r\nQ 35.15625 65.921875 28.609375 65.921875 \r\nQ 23.96875 65.921875 18.8125 64.3125 \r\nQ 13.671875 62.703125 7.8125 59.421875 \r\nL 7.8125 69.390625 \r\nQ 13.765625 71.78125 18.9375 73 \r\nQ 24.125 74.21875 28.421875 74.21875 \r\nQ 39.75 74.21875 46.484375 68.546875 \r\nQ 53.21875 62.890625 53.21875 53.421875 \r\nQ 53.21875 48.921875 51.53125 44.890625 \r\nQ 49.859375 40.875 45.40625 35.40625 \r\nQ 44.1875 33.984375 37.640625 27.21875 \r\nQ 31.109375 20.453125 19.1875 8.296875 \r\nz\r\n\" id=\"DejaVuSans-50\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_3\">\r\n     <g id=\"line2d_5\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"39.65\" xlink:href=\"#m80e23da729\" y=\"131.672227\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_5\">\r\n      <!-- 4000 -->\r\n      <g transform=\"translate(7.2 135.471446)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 37.796875 64.3125 \r\nL 12.890625 25.390625 \r\nL 37.796875 25.390625 \r\nz\r\nM 35.203125 72.90625 \r\nL 47.609375 72.90625 \r\nL 47.609375 25.390625 \r\nL 58.015625 25.390625 \r\nL 58.015625 17.1875 \r\nL 47.609375 17.1875 \r\nL 47.609375 0 \r\nL 37.796875 0 \r\nL 37.796875 17.1875 \r\nL 4.890625 17.1875 \r\nL 4.890625 26.703125 \r\nz\r\n\" id=\"DejaVuSans-52\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-52\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_4\">\r\n     <g id=\"line2d_6\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"39.65\" xlink:href=\"#m80e23da729\" y=\"85.188341\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_6\">\r\n      <!-- 6000 -->\r\n      <g transform=\"translate(7.2 88.987559)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 33.015625 40.375 \r\nQ 26.375 40.375 22.484375 35.828125 \r\nQ 18.609375 31.296875 18.609375 23.390625 \r\nQ 18.609375 15.53125 22.484375 10.953125 \r\nQ 26.375 6.390625 33.015625 6.390625 \r\nQ 39.65625 6.390625 43.53125 10.953125 \r\nQ 47.40625 15.53125 47.40625 23.390625 \r\nQ 47.40625 31.296875 43.53125 35.828125 \r\nQ 39.65625 40.375 33.015625 40.375 \r\nz\r\nM 52.59375 71.296875 \r\nL 52.59375 62.3125 \r\nQ 48.875 64.0625 45.09375 64.984375 \r\nQ 41.3125 65.921875 37.59375 65.921875 \r\nQ 27.828125 65.921875 22.671875 59.328125 \r\nQ 17.53125 52.734375 16.796875 39.40625 \r\nQ 19.671875 43.65625 24.015625 45.921875 \r\nQ 28.375 48.1875 33.59375 48.1875 \r\nQ 44.578125 48.1875 50.953125 41.515625 \r\nQ 57.328125 34.859375 57.328125 23.390625 \r\nQ 57.328125 12.15625 50.6875 5.359375 \r\nQ 44.046875 -1.421875 33.015625 -1.421875 \r\nQ 20.359375 -1.421875 13.671875 8.265625 \r\nQ 6.984375 17.96875 6.984375 36.375 \r\nQ 6.984375 53.65625 15.1875 63.9375 \r\nQ 23.390625 74.21875 37.203125 74.21875 \r\nQ 40.921875 74.21875 44.703125 73.484375 \r\nQ 48.484375 72.75 52.59375 71.296875 \r\nz\r\n\" id=\"DejaVuSans-54\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-54\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_5\">\r\n     <g id=\"line2d_7\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"39.65\" xlink:href=\"#m80e23da729\" y=\"38.704454\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_7\">\r\n      <!-- 8000 -->\r\n      <g transform=\"translate(7.2 42.503673)scale(0.1 -0.1)\">\r\n       <defs>\r\n        <path d=\"M 31.78125 34.625 \r\nQ 24.75 34.625 20.71875 30.859375 \r\nQ 16.703125 27.09375 16.703125 20.515625 \r\nQ 16.703125 13.921875 20.71875 10.15625 \r\nQ 24.75 6.390625 31.78125 6.390625 \r\nQ 38.8125 6.390625 42.859375 10.171875 \r\nQ 46.921875 13.96875 46.921875 20.515625 \r\nQ 46.921875 27.09375 42.890625 30.859375 \r\nQ 38.875 34.625 31.78125 34.625 \r\nz\r\nM 21.921875 38.8125 \r\nQ 15.578125 40.375 12.03125 44.71875 \r\nQ 8.5 49.078125 8.5 55.328125 \r\nQ 8.5 64.0625 14.71875 69.140625 \r\nQ 20.953125 74.21875 31.78125 74.21875 \r\nQ 42.671875 74.21875 48.875 69.140625 \r\nQ 55.078125 64.0625 55.078125 55.328125 \r\nQ 55.078125 49.078125 51.53125 44.71875 \r\nQ 48 40.375 41.703125 38.8125 \r\nQ 48.828125 37.15625 52.796875 32.3125 \r\nQ 56.78125 27.484375 56.78125 20.515625 \r\nQ 56.78125 9.90625 50.3125 4.234375 \r\nQ 43.84375 -1.421875 31.78125 -1.421875 \r\nQ 19.734375 -1.421875 13.25 4.234375 \r\nQ 6.78125 9.90625 6.78125 20.515625 \r\nQ 6.78125 27.484375 10.78125 32.3125 \r\nQ 14.796875 37.15625 21.921875 38.8125 \r\nz\r\nM 18.3125 54.390625 \r\nQ 18.3125 48.734375 21.84375 45.5625 \r\nQ 25.390625 42.390625 31.78125 42.390625 \r\nQ 38.140625 42.390625 41.71875 45.5625 \r\nQ 45.3125 48.734375 45.3125 54.390625 \r\nQ 45.3125 60.0625 41.71875 63.234375 \r\nQ 38.140625 66.40625 31.78125 66.40625 \r\nQ 25.390625 66.40625 21.84375 63.234375 \r\nQ 18.3125 60.0625 18.3125 54.390625 \r\nz\r\n\" id=\"DejaVuSans-56\"/>\r\n       </defs>\r\n       <use xlink:href=\"#DejaVuSans-56\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"127.246094\" xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"190.869141\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"patch_5\">\r\n    <path d=\"M 39.65 224.64 \r\nL 39.65 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_6\">\r\n    <path d=\"M 374.45 224.64 \r\nL 374.45 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_7\">\r\n    <path d=\"M 39.65 224.64 \r\nL 374.45 224.64 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_8\">\r\n    <path d=\"M 39.65 7.2 \r\nL 374.45 7.2 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n  </g>\r\n </g>\r\n <defs>\r\n  <clipPath id=\"p9c207e56d7\">\r\n   <rect height=\"217.44\" width=\"334.8\" x=\"39.65\" y=\"7.2\"/>\r\n  </clipPath>\r\n </defs>\r\n</svg>\r\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAQjElEQVR4nO3ce7BeVX3G8e9jIuClAkpEG9IeRtNSbKtiClhaq9LhIp0GrRdaK0HpZJzidYoK1ineGPFW0FHRVMDgWAEpCAIjk4I4OBYkXESBMqRcJBE0mgBeBjTw6x/vCr7Gc3LeAyfnJK7vZ+ads/baa++99snOs9e73n3eVBWSpD48ZrY7IEmaOYa+JHXE0Jekjhj6ktQRQ1+SOjJ3tjuwObvsskuNjY3NdjckaZty9dVX/6iq5o23bqsO/bGxMVauXDnb3ZCkbUqSOyZa5/SOJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1ZKv+i9xHa+yYC2e7C9pK3X7CIbPdBWlWONKXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjowU+knemuSGJN9N8sUkOyTZPcmVSVYlOTPJdq3t9m15VVs/NrSfY1v9zUkO3ELnJEmawKShn2Q+8CZgUVX9MTAHOAz4IHBiVT0TWA8c2TY5Eljf6k9s7UiyZ9vuWcBBwKeSzJne05Ekbc6o0ztzgcclmQs8HrgLeDFwdlu/HDi0lRe3Zdr6/ZOk1Z9RVQ9U1W3AKmDvR30GkqSRTRr6VbUG+AjwPQZhfy9wNXBPVW1ozVYD81t5PnBn23ZDa/+U4fpxtnlYkqVJViZZuXbt2kdyTpKkCYwyvbMzg1H67sDvAk9gMD2zRVTVsqpaVFWL5s2bt6UOI0ldGmV656+B26pqbVX9EjgH2A/YqU33AOwGrGnlNcACgLZ+R+DHw/XjbCNJmgGjhP73gH2TPL7Nze8P3Ah8DXh5a7MEOK+Vz2/LtPWXVlW1+sPa0z27AwuBb03PaUiSRjF3sgZVdWWSs4FrgA3AtcAy4ELgjCTvb3WntE1OAT6fZBWwjsETO1TVDUnOYnDD2AAcVVUPTvP5SJI2Y9LQB6iq44DjNqm+lXGevqmq+4FXTLCf44Hjp9hHSdI08S9yJakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHRvpqZUlbxtgxF852F7SVuv2EQ7bIfh3pS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0JekjowU+kl2SnJ2kv9NclOS5yd5cpIVSW5pP3dubZPk40lWJbk+yV5D+1nS2t+SZMmWOilJ0vhGHel/DPhqVe0BPBu4CTgGuKSqFgKXtGWAg4GF7bUUOBkgyZOB44B9gL2B4zbeKCRJM2PS0E+yI/AC4BSAqvpFVd0DLAaWt2bLgUNbeTFweg1cAeyU5OnAgcCKqlpXVeuBFcBB03gukqRJjDLS3x1YC5yW5Nokn03yBGDXqrqrtbkb2LWV5wN3Dm2/utVNVP9rkixNsjLJyrVr107tbCRJmzVK6M8F9gJOrqrnAj/jV1M5AFRVATUdHaqqZVW1qKoWzZs3bzp2KUlqRgn91cDqqrqyLZ/N4CbwgzZtQ/v5w7Z+DbBgaPvdWt1E9ZKkGTJp6FfV3cCdSf6wVe0P3AicD2x8AmcJcF4rnw8c3p7i2Re4t00DXQwckGTn9gHuAa1OkjRD5o7Y7o3AF5JsB9wKvJbBDeOsJEcCdwCvbG0vAl4CrAJ+3tpSVeuSvA+4qrV7b1Wtm5azkCSNZKTQr6rrgEXjrNp/nLYFHDXBfk4FTp1C/yRJ08i/yJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHRk59JPMSXJtkgva8u5JrkyyKsmZSbZr9du35VVt/djQPo5t9TcnOXDaz0aStFlTGem/GbhpaPmDwIlV9UxgPXBkqz8SWN/qT2ztSLIncBjwLOAg4FNJ5jy67kuSpmKk0E+yG3AI8Nm2HODFwNmtyXLg0FZe3JZp6/dv7RcDZ1TVA1V1G7AK2HsazkGSNKJRR/onAW8HHmrLTwHuqaoNbXk1ML+V5wN3ArT197b2D9ePs83DkixNsjLJyrVr145+JpKkSU0a+kn+BvhhVV09A/2hqpZV1aKqWjRv3ryZOKQkdWPuCG32A/42yUuAHYAnAR8Ddkoyt43mdwPWtPZrgAXA6iRzgR2BHw/VbzS8jSRpBkw60q+qY6tqt6oaY/BB7KVV9Wrga8DLW7MlwHmtfH5bpq2/tKqq1R/Wnu7ZHVgIfGvazkSSNKlRRvoTeQdwRpL3A9cCp7T6U4DPJ1kFrGNwo6CqbkhyFnAjsAE4qqoefBTHlyRN0ZRCv6ouAy5r5VsZ5+mbqrofeMUE2x8PHD/VTkqSpod/kStJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRSUM/yYIkX0tyY5Ibkry51T85yYokt7SfO7f6JPl4klVJrk+y19C+lrT2tyRZsuVOS5I0nlFG+huAf6mqPYF9gaOS7AkcA1xSVQuBS9oywMHAwvZaCpwMg5sEcBywD7A3cNzGG4UkaWZMGvpVdVdVXdPKPwFuAuYDi4Hlrdly4NBWXgycXgNXADsleTpwILCiqtZV1XpgBXDQdJ6MJGnzpjSnn2QMeC5wJbBrVd3VVt0N7NrK84E7hzZb3eomqt/0GEuTrEyycu3atVPpniRpEiOHfpInAv8FvKWq7hteV1UF1HR0qKqWVdWiqlo0b9686dilJKkZKfSTPJZB4H+hqs5p1T9o0za0nz9s9WuABUOb79bqJqqXJM2QUZ7eCXAKcFNV/fvQqvOBjU/gLAHOG6o/vD3Fsy9wb5sGuhg4IMnO7QPcA1qdJGmGzB2hzX7Aa4DvJLmu1b0TOAE4K8mRwB3AK9u6i4CXAKuAnwOvBaiqdUneB1zV2r23qtZNx0lIkkYzaehX1TeATLB6/3HaF3DUBPs6FTh1Kh2UJE0f/yJXkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHVkxkM/yUFJbk6yKskxM318SerZjIZ+kjnAJ4GDgT2Bv0+y50z2QZJ6NtMj/b2BVVV1a1X9AjgDWDzDfZCkbs2d4ePNB+4cWl4N7DPcIMlSYGlb/GmSm2eob7/tdgF+NNud2Frkg7PdA43Da3TIo7xGf3+iFTMd+pOqqmXAstnux2+bJCuratFs90OaiNfozJjp6Z01wIKh5d1anSRpBsx06F8FLEyye5LtgMOA82e4D5LUrRmd3qmqDUneAFwMzAFOraobZrIPHXPKTFs7r9EZkKqa7T5IkmaIf5ErSR0x9CWpI4b+LEny0iTXbfJ6KMnBW+BYT0tyRpL/S3J1kouS/ME0H+OFSf58OvepLSNJJfno0PLRSd49Tfu+eJNr+vtJrpyOfY9zrMOTfDfJd5Jcm+ToLXCMd073PmeboT9LqurcqnrOxhfwKeByBh9yTyoDk/77JQlwLnBZVT2jqp4HHAvs+sh7P64XAob+tuEB4GVJdpnuHVfVgUPX9H7AfcC7Rt0+yUgPl7TB0VuAA6rqT4B9gXun3OHJGfqafm3U/W/Aa6rqoVb3tiRXJbk+yXta3Vj7srrTge8CC5J8eGi086pxdv8i4JdV9emNFVX17aq6vN04fmP7Nmq/YKh/n0hyRCvfnuQ9Sa5p2+yRZAx4PfDWNrr7yy3xe9K02cDgSZm3brqiXWOXtuvukiS/1+o/l+TjSb6Z5NYkLx/hOB8DLqqqFW0fz0jy1fZu8/Ikewzt+9PtHcGHkjwnyRWtD+cm2XmcfR8LHF1V3weoqgeq6j/a/sbdPsllSRa18i5Jbm/lI5Kc0/p2S5IPtfoTgMe1a/oLI/92t3ZV5WsWX8BjgZXAq4bqDmDwnzIMbswXAC8AxoCHgH1bu78DVjB4/HVX4HvA0zfZ/5uAEyc49rjbMxi1XzDU7hPAEa18O/DGVv5n4LOt/G4G/wln/Xfqa9Jr7qfAk9q/5Y7A0cC727qvAEta+XXAl1v5c8CX2vW4J4Pv0NrcMV4GXAdsP1R3CbCwlfcBLh3a9wXAnLZ8PfBXrfxe4KRx9r8O2HGCY4+7PXAZsKiVdwFub+UjgFvb72IH4A5gwcbf1Wz/e033y5H+7HsfcENVnTlUd0B7XQtcA+wBLGzr7qiqK1r5L4AvVtWDVfUD4OvAn03h2I90+3Paz6sZ3Ii0jamq+4DTGQwKhj0f+M9W/jyDa2SjL1fVQ1V1I5uZHkwyn8Eo/x+q6oFW90QG039fSnId8BkGA4yNvlRVDybZEdipqr7e6pczGPCM5FFsf0lV3VtV9wM3spnvrtnWbXXfvdOTJC9kMNrea9NVwAeq6jObtB8DfjbFw9wAjPJWfNgGfn3qb4dN1j/Qfj6I19C27CQGg4rTRmz/wFA5AEmOBw4BqKrntM+QlgMntJvDRo8B7qnBXP94Hsl1/Tzg0ilsM3xdT3RNw2/5de1If5a0ecbTgMOr6iebrL4YeF0bHZFkfpKnjrOby4FXJZmTZB6DEc23NmlzKbB9Bt9euvHYf9rm3Sfa/g5gzyTbJ9kJ2H+EU/oJ8DsjtNNWoqrWAWcBRw5Vf5PB16MAvJrBNbK5ffxr/eqDWxhMFd1fVZ/cpN19wG1JXgEPP4jw7HH2dy+wfuhzodcweAe6qQ8AH07ytLa/7ZL80yTb387gRgGjD4R+meSxI7bdJvzW3s22Aa8HngqcPBgcPewDVXVmkj8C/qet+ynwjwxGIMPOZfB2/NtAAW+vqruHG1RVJXkpcFKSdwD3M7j43wJ8Y6Ltk5zF4MPi2xhMM03mK8DZSRYzmPPfbFhoq/FR4A1Dy28ETkvyNmAt8Nop7u/9wOo2hbPR+qp6EYObyMlJ3sXgs6wzGFx7m1oCfDrJ4xnMtf9GH6rqoiS7Av/d3l0UcOok238EOKsNgC4c8XyWAdcnuaaqXj3iNls1v4ZBkjri9I4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR35f4+cvUgdYTxzAAAAAElFTkSuQmCC\n"
     },
     "metadata": {
      "needs_background": "light"
     }
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "#First of all I need to take the data and change it to have a class name, instead of continous, this will help for sampling and classifier later, 1 is greater than 0\n",
    "claim_data_selected[\"Binary\"] = np.where(claim_data_selected[\"Claim_Amount\"] > 0, 1,0) \n",
    "\n",
    "#Then split the data to have a test set that I can use when I have my ideal model\n",
    "claim_data_train, claim_data_test = train_test_split(claim_data_selected, test_size=0.2, random_state=47)\n",
    "\n",
    "#Then I need to create a validation set for my training data\n",
    "claim_data_train2, claim_data_valid = train_test_split(claim_data_train, test_size=0.2, random_state=47)\n",
    "\n",
    "#I do everything twice here, which could be negated with h and vstack, but I work better this way personally\n",
    "#I am fist going to undersample the data\n",
    "claim_data_train_under = sampler(claim_data_train)\n",
    "claim_data_train2_under = sampler(claim_data_train2)\n",
    "\n",
    "#Create variables for later use before the drop\n",
    "claim_data_test_bin = claim_data_test\n",
    "claim_data_valid_bin = claim_data_valid\n",
    "claim_data_train_under_bin = claim_data_train_under\n",
    "claim_data_train2_under_bin = claim_data_train2_under\n",
    "\n",
    "#Drop some values for there use later in the classifier\n",
    "claim_data_train_under_bin = claim_data_train_under_bin.drop([\"Claim_Amount\"], axis = 1)\n",
    "claim_data_train2_under_bin = claim_data_train2_under_bin.drop([\"Claim_Amount\"], axis = 1)\n",
    "claim_data_test_bin = claim_data_test_bin.drop([\"Claim_Amount\"], axis = 1)\n",
    "claim_data_valid_bin = claim_data_valid_bin.drop([\"Claim_Amount\"], axis = 1)\n",
    "\n",
    "#Drop the added columns going forwards as they are only needed in classifcation\n",
    "claim_data_train_under = claim_data_train_under.drop([\"Binary\"], axis = 1)\n",
    "claim_data_train2_under = claim_data_train2_under.drop([\"Binary\"], axis = 1)\n",
    "claim_data_test = claim_data_test.drop([\"Binary\"], axis = 1)\n",
    "claim_data_valid = claim_data_valid.drop([\"Binary\"], axis = 1)\n",
    "\n",
    "#Check it worked \n",
    "%matplotlib inline \n",
    "import matplotlib.pyplot as plt\n",
    "zero_count = claim_data_train_under[(claim_data_train_under[\"Claim_Amount\"] == 0)].count()[0]\n",
    "non_zero_count = claim_data_train_under[claim_data_train_under[\"Claim_Amount\" ] > 0].count()[0]\n",
    "print(zero_count)\n",
    "print(non_zero_count)\n",
    "plt.bar([\"Zero Count\", \"Non-Zero Count\"], [zero_count, non_zero_count])\n",
    "\n",
    "#Create the training sets with just 0 values in them\n",
    "claim_data_train_under_nonzero = claim_data_train_under[claim_data_train_under[\"Claim_Amount\"] > 0]\n",
    "claim_data_train2_under_nonzero = claim_data_train2_under[claim_data_train2_under[\"Claim_Amount\"] > 0]\n",
    "\n",
    "#I now need to split the data into features and labels now it is sampled, first full train and test sets\n",
    "claim_data_train_under_features = claim_data_train_under.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_train_under_labels = pd.DataFrame(claim_data_train_under[\"Claim_Amount\"])\n",
    "claim_data_test_features = claim_data_test.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_test_labels = pd.DataFrame(claim_data_test[\"Claim_Amount\"])\n",
    "\n",
    "#Now do my train2 and validation split\n",
    "claim_data_train2_under_features = claim_data_train2_under.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_train2_under_labels = pd.DataFrame(claim_data_train2_under[\"Claim_Amount\"])\n",
    "claim_data_valid_features = claim_data_valid.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_valid_labels = pd.DataFrame(claim_data_valid[\"Claim_Amount\"])\n",
    "\n",
    "#I now need to perform the data pre processing above and change the data over using my column transformer.\n",
    "claim_data_train_under_features_transformed = tree_transformer.fit_transform(claim_data_train_under_features)\n",
    "claim_data_test_features_transformed = tree_transformer.transform(claim_data_test_features)\n",
    "claim_data_train2_under_features_transformed = full_transform.fit_transform(claim_data_train2_under_features)\n",
    "claim_data_valid_features_transformed = full_transform.transform(claim_data_valid_features)\n",
    "\n",
    "#Now I will peform a grid search with these features\n",
    "lr = LinearRegression()\n",
    "#parameters = dict(fit_intercept = ['True', 'False'])\n",
    "#grid = GridSearchCV(lr, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_features_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "lr_best = {'fit_intercept': 'True'}\n",
    "\n",
    "#Set the optimal model hyper params\n",
    "lr.set_params(**lr_best)\n",
    "\n",
    "#Fit the model with the training data\n",
    "lr.fit(claim_data_train2_under_features_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = lr.predict(claim_data_valid_features_transformed)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(claim_data_valid_labels, pred)\n",
    "print(\"The MSE for Linear Regression is\",error)"
   ]
  },
  {
   "source": [
    "Question 2 - Peformance Using a single model - Ridge Regression"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The MSE for Ridge Regression is 49045.32999909534\n"
     ]
    }
   ],
   "source": [
    "#Use the variables we created above now the data is fully ready and transformed\n",
    "from sklearn.linear_model import Ridge\n",
    "ridge = Ridge()\n",
    "#First set up the grid search again\n",
    "#parameters = dict(fit_intercept = ['True', 'False'], alpha = [20,10,5,1,0.1,0.01,0.001,0.0001,0], solver = ['svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'])\n",
    "#grid = GridSearchCV(ridge, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_features_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "#The best params from the grid search\n",
    "ridge_best = {'alpha': 20, 'fit_intercept': 'False', 'solver': 'saga'}\n",
    "\n",
    "#Set the best params\n",
    "ridge.set_params(**ridge_best)\n",
    "\n",
    "#fit the model\n",
    "ridge.fit(claim_data_train2_under_features_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = ridge.predict(claim_data_valid_features_transformed)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(claim_data_valid_labels, pred)\n",
    "print(\"The MSE for Ridge Regression is\",error)"
   ]
  },
  {
   "source": [
    "Question 2 - Peformance Using a single model - Random Forests Regression"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The MSE for Random Forest Regression is 49053.162065855315\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "#We now need to appy a different transformer as the numerical data does not need it\n",
    "claim_data_train2_under_features_tree_transformed = tree_transformer.fit_transform(claim_data_train2_under_features)\n",
    "claim_data_valid_features_tree_transformed = tree_transformer.transform(claim_data_valid_features)\n",
    "\n",
    "rf = RandomForestRegressor()\n",
    "\n",
    "#Grid search\n",
    "#parameters = dict(n_estimators = [20, 50, 100, 200], max_samples = [None,500, 1500, 3000], max_depth=[None,3,5,7,9])\n",
    "#grid = GridSearchCV(rf, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_features_tree_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "rf_best = {'max_depth': 3, 'max_samples': None, 'n_estimators': 200}\n",
    "\n",
    "#Set the best params\n",
    "rf.set_params(**rf_best)\n",
    "\n",
    "#fit the model\n",
    "rf.fit(claim_data_train2_under_features_tree_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = rf.predict(claim_data_valid_features_tree_transformed)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(claim_data_valid_labels, pred)\n",
    "print(\"The MSE for Random Forest Regression is\",error)"
   ]
  },
  {
   "source": [
    "Question 2 - Peformance Using a single model - Gradient Tree Boosting Regression"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The MSE for Gradient Boosting Regression is 48737.06859474875\n"
     ]
    }
   ],
   "source": [
    "## import GradientBoostingRegressor package\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "gbr = GradientBoostingRegressor()\n",
    "\n",
    "#Grid search\n",
    "#parameters = dict(n_estimators = [20, 50, 100, 200], max_features = ['auto', 'sqrt','log2'] ,max_depth=[3,5,7,9], learning_rate = [0.1,0.01,0.25,0.5])\n",
    "#grid = GridSearchCV(gbr, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_features_tree_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "gbr_best = {'learning_rate': 0.01, 'max_depth': 3, 'max_features': 'sqrt', 'n_estimators': 100}\n",
    "\n",
    "#Set the best params\n",
    "gbr.set_params(**gbr_best)\n",
    "\n",
    "#fit the model\n",
    "gbr.fit(claim_data_train2_under_features_tree_transformed, claim_data_train2_under_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = gbr.predict(claim_data_valid_features_tree_transformed)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(claim_data_valid_labels, pred)\n",
    "print(\"The MSE for Gradient Boosting Regression is\",error)"
   ]
  },
  {
   "source": [
    "The above runs have shown Gradient Boosting to pedict with the lowest MSE on our validation set, showing that this is the ideal single model. The other thing to note is the outperfomance of Ridge Regression on Linear, with that being the highest non tree regressor. In future runs using a single model I will use GBR as my model."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Question 3 - Performance Using A Combination of Two Models - Part A\n",
    "\n",
    "The first model will be a binary classifier that will tell whether the claim was zero or different from zero. Compare the following classifiers: random forests for classification and gradient boosting for classification [3 marks]."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "First I will create a classifier model, and I will compare Random Forest and Gradient Boost again. I have selected F-Score as a way of comparing these models to see which is best. I believe F-Score is a good way to measure the effectiveness of the model here due to the imbalance still found in the dataset (even with it being small. This harmonic mean of precison and recall will be a more accurate way of reflecting how well the classifier is predicting opposed to stanadard accuracy which can be misleading. What ever is the best classifier I will take these predictions into the second model, and if the prediction belongs to the greater than 0 class I will then perform regression on that using a special training set with just greater than 0 values on. I will then combine the predictions of this model with what it has said is 0 and the regression predictions to calculate my MSE"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The RFC score is:  0.30928764652840396\n",
      "The GBC score is:  0.33031479085812854\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "#Remind us of the variables we have above\n",
    "claim_data_train_under_bin\n",
    "claim_data_train2_under_bin\n",
    "claim_data_test_bin\n",
    "claim_data_valid_bin\n",
    "\n",
    "#Use these variables to set the correct target classes etc\n",
    "#I now need to split the data into features and labels now it is sampled, first full train and test sets\n",
    "claim_data_train_under_bin_features = claim_data_train_under_bin.drop([\"Binary\"], axis=1)\n",
    "claim_data_train_under_bin_labels = pd.DataFrame(claim_data_train_under_bin[\"Binary\"])\n",
    "claim_data_test_bin_features = claim_data_test_bin.drop([\"Binary\"], axis=1)\n",
    "claim_data_test_bin_labels = pd.DataFrame(claim_data_test_bin[\"Binary\"])\n",
    "\n",
    "#Now do my train2 and validation split\n",
    "claim_data_train2_under_bin_features = claim_data_train2_under_bin.drop([\"Binary\"], axis=1)\n",
    "claim_data_train2_under_bin_labels = pd.DataFrame(claim_data_train2_under_bin[\"Binary\"])\n",
    "claim_data_valid_bin_features = claim_data_valid_bin.drop([\"Binary\"], axis=1)\n",
    "claim_data_valid_bin_labels = pd.DataFrame(claim_data_valid_bin[\"Binary\"])\n",
    "\n",
    "#We now need to appy a different transformer as the numerical data does not need it\n",
    "claim_data_train2_under_bin_features_tree_transformed = tree_transformer.fit_transform(claim_data_train2_under_bin_features)\n",
    "claim_data_valid_bin_features_tree_transformed = tree_transformer.transform(claim_data_valid_bin_features)\n",
    "\n",
    "#Try out the RFC\n",
    "rfc = RandomForestClassifier()\n",
    "\n",
    "#Grid search\n",
    "#parameters = dict(n_estimators = [20, 50, 100, 200], max_samples = [None,500, 1500, 3000], max_depth=[None,3,5,7,9])\n",
    "#grid = GridSearchCV(rfc, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_bin_features_tree_transformed, claim_data_train2_under_bin_labels)\n",
    "\n",
    "rfc_best = {'max_depth': 9, 'max_samples': None, 'n_estimators': 100}\n",
    "\n",
    "#Set the best params\n",
    "rfc.set_params(**rfc_best)\n",
    "\n",
    "#fit the model\n",
    "rfc.fit(claim_data_train2_under_bin_features_tree_transformed, claim_data_train2_under_bin_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "rfc_pred = rfc.predict(claim_data_valid_bin_features_tree_transformed)\n",
    "\n",
    "print(\"The RFC score is: \", f1_score(claim_data_valid_bin_labels,rfc_pred))\n",
    "\n",
    "#Try out the GBC\n",
    "gbc = GradientBoostingClassifier()\n",
    "\n",
    "#Grid search\n",
    "#parameters = dict(n_estimators = [20, 50, 100, 200], max_features = ['auto', 'sqrt','log2'] ,max_depth=[3,5,7,9], learning_rate = [0.1,0.01,0.25,0.5])\n",
    "#grid = GridSearchCV(gbc, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_bin_features_tree_transformed, claim_data_train2_under_bin_labels)\n",
    "\n",
    "gbc_best = {'learning_rate': 0.1, 'max_depth': 5, 'max_features': 'sqrt', 'n_estimators': 50}\n",
    "\n",
    "#Set the best params\n",
    "gbc.set_params(**gbc_best)\n",
    "\n",
    "#fit the model\n",
    "gbc.fit(claim_data_train2_under_bin_features_tree_transformed, claim_data_train2_under_bin_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "gbc_pred = gbc.predict(claim_data_valid_bin_features_tree_transformed)\n",
    "\n",
    "print(\"The GBC score is: \", f1_score(claim_data_valid_bin_labels,gbc_pred))"
   ]
  },
  {
   "source": [
    "Question 3 - Performance Using A Combination of Two Models - Part B\n",
    "\n",
    "For the second model, if the claim was different from zero, train a regression model to predict the actual value of the claim. Compare the same models that you used in step 2 [3 marks]."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The MSE of the dual model is 52963.11095638728\n"
     ]
    }
   ],
   "source": [
    "#Lets use our best regressor from above which is GBR\n",
    "gbrQ3 = GradientBoostingRegressor()\n",
    "\n",
    "#We need to make the dataset for training just include non 0 values, I already created these above\n",
    "claim_data_train2_under_nonzero\n",
    "\n",
    "#Now do my train2 and validation split\n",
    "claim_data_train2_under_nonzero_features = claim_data_train2_under_nonzero.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_train2_under_nonzero_labels = pd.DataFrame(claim_data_train2_under_nonzero[\"Claim_Amount\"])\n",
    "\n",
    "#Finally transform my features\n",
    "claim_data_train2_under_nonzero_features_tree_transformed = tree_transformer.fit_transform(claim_data_train2_under_nonzero_features)\n",
    "\n",
    "#I also need my validation to be just 0, take the labels with value and combine with predictions, order should be equal as has not been shuffled\n",
    "predicted_actual = np.hstack((claim_data_valid_labels, gbc_pred.reshape(-1,1)))\n",
    "\n",
    "#Then I need to combie in the features\n",
    "features_predicted_actual = np.hstack((claim_data_valid_bin_features_tree_transformed,predicted_actual))\n",
    "\n",
    "#And finally seperate the new validation set into features and labels\n",
    "new_valid_features = []\n",
    "new_valid_labels = []\n",
    "new_valid_labels_zero = []\n",
    "new_full_labels_zero = []\n",
    "\n",
    "for row in features_predicted_actual:\n",
    "    #If predicted greater than 0, add it to the new validation set\n",
    "    if row[-1] == 1:\n",
    "        new_valid_features.append(row[0:-2])\n",
    "        new_valid_labels.append(row[-2])\n",
    "    #Else just add a 0 to a list as we have said it has 0 from model A, also store the real label value in another array\n",
    "    if row[-1] == 0:\n",
    "        new_full_labels_zero.append(row[-2])\n",
    "        new_valid_labels_zero.append(0)\n",
    "\n",
    "#Grid search\n",
    "#parameters = dict(n_estimators = [20, 50, 100, 200], max_features = ['auto', 'sqrt','log2'] ,max_depth=[3,5,7,9], learning_rate = [0.1,0.01,0.25,0.5])\n",
    "#grid = GridSearchCV(gbr, param_grid=parameters, cv=5, n_jobs=-1, scoring='neg_mean_squared_error')\n",
    "#grid.fit(claim_data_train2_under_nonzero_features_tree_transformed, claim_data_train2_under_nonzero_labels)\n",
    "\n",
    "#Set the best params\n",
    "gbrQ3.set_params(**gbr_best)\n",
    "\n",
    "#fit the model\n",
    "gbrQ3.fit(claim_data_train2_under_nonzero_features_tree_transformed, claim_data_train2_under_nonzero_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = gbrQ3.predict(new_valid_features)\n",
    "\n",
    "#Change the pred to a list\n",
    "pred = pred.tolist()\n",
    "\n",
    "#Add the zero labels we created earlier (modelA predictions) to our list, pred now has our entire validation set predictions\n",
    "pred.extend(new_valid_labels_zero)\n",
    "\n",
    "#Add our validation labels to the real labels modelA said was 0, this is now the original full list for the validation labels\n",
    "new_valid_labels.extend(new_full_labels_zero)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(new_valid_labels, pred)\n",
    "print(\"The MSE of the dual model is\", error)"
   ]
  },
  {
   "source": [
    "The result of this having a higer MSE was suprinsing at first but on consideration it makes some sense. I believe the classifier is very innacurate as reflected in it's ~0.3 F-Score value. This indicates to me a lot of data validating the second model is totally wrong. Also with a single model it's more likely a value is to be predicted at a small value than 0 if it does have a claim value, where as here if it is misclassified by model A it has to be 0, increasing the distance from the answer."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Question 4 - Report the performance of the best models over the test set [2 marks].\n",
    "\n",
    "Compute the performance metric over the test set for the best model in Step 2  and the best model in Step 3."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The MSE for step 2 is 63433.475955841895\n",
      "The MSE for step 3 is 68007.5711701075\n"
     ]
    }
   ],
   "source": [
    "#The best model found in question two is the gradientbosstingregressor, so lets take the model and give it the test data\n",
    "gbrQ4 = GradientBoostingRegressor()\n",
    "\n",
    "#Our last best params are also a GBR\n",
    "gbrQ4.set_params(**gbr_best)\n",
    "\n",
    "#Train on the whole dataset\n",
    "gbrQ4.fit(claim_data_train_under_features_transformed, claim_data_train_under_labels)\n",
    "\n",
    "#Predict on our test set\n",
    "pred = gbrQ4.predict(claim_data_test_features_transformed)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(claim_data_test_labels, pred)\n",
    "print(\"The MSE for step 2 is\", error)\n",
    "\n",
    "################################################################################\n",
    "#Now lets do from Q3\n",
    "gbc = GradientBoostingClassifier()\n",
    "\n",
    "#Tree transform the \n",
    "claim_data_train_under_bin_features_tree_transformed = tree_transformer.fit_transform(claim_data_train_under_bin_features)\n",
    "claim_data_test_bin_features_tree_transformed = tree_transformer.transform(claim_data_test_bin_features)\n",
    "\n",
    "#Set the best params\n",
    "gbc.set_params(**gbc_best)\n",
    "\n",
    "#fit the model\n",
    "gbc.fit(claim_data_train_under_bin_features_tree_transformed, claim_data_train_under_bin_labels)\n",
    "\n",
    "#Predict on the test set\n",
    "gbc_pred = gbc.predict(claim_data_test_bin_features_tree_transformed)\n",
    "\n",
    "#Lets use our best regressor from above which is GBR\n",
    "gbc = GradientBoostingRegressor()\n",
    "\n",
    "#We need to make the dataset for training just include non 0 values, I already created these above\n",
    "claim_data_train_under_nonzero\n",
    "\n",
    "#Now do my train and test split\n",
    "claim_data_train_under_nonzero_features = claim_data_train2_under_nonzero.drop([\"Claim_Amount\"], axis=1)\n",
    "claim_data_train_under_nonzero_labels = pd.DataFrame(claim_data_train2_under_nonzero[\"Claim_Amount\"])\n",
    "\n",
    "#Finally transform my features\n",
    "claim_data_train_under_nonzero_features_tree_transformed = tree_transformer.fit_transform(claim_data_train_under_nonzero_features)\n",
    "\n",
    "#I also need my validation to be just 0, take the labels with value and combine with predictions, order should be equal as has not been shuffled\n",
    "predicted_actual = np.hstack((claim_data_test_labels, gbc_pred.reshape(-1,1)))\n",
    "\n",
    "#Then I need to combie in the features\n",
    "features_predicted_actual = np.hstack((claim_data_test_bin_features_tree_transformed,predicted_actual))\n",
    "\n",
    "#And finally seperate the new validation set into features and labels\n",
    "new_valid_features = []\n",
    "new_valid_labels = []\n",
    "new_valid_labels_zero = []\n",
    "new_full_labels_zero = []\n",
    "\n",
    "for row in features_predicted_actual:\n",
    "    #If predicted greater than 0, add it to the new validation set\n",
    "    if row[-1] == 1:\n",
    "        new_valid_features.append(row[0:-2])\n",
    "        new_valid_labels.append(row[-2])\n",
    "    #Else just add a 0 to a list as we have said it has 0 from model A, also store the real label value in another array\n",
    "    if row[-1] == 0:\n",
    "        new_full_labels_zero.append(row[-2])\n",
    "        new_valid_labels_zero.append(0)\n",
    "#Set the best params\n",
    "gbc.set_params(**gbr_best)\n",
    "\n",
    "#fit the model\n",
    "gbc.fit(claim_data_train_under_nonzero_features_tree_transformed, claim_data_train_under_nonzero_labels)\n",
    "\n",
    "#Predict on the validation set\n",
    "pred = gbc.predict(new_valid_features)\n",
    "\n",
    "#Change the pred to a list\n",
    "pred = pred.tolist()\n",
    "\n",
    "#Add the zero labels we created earlier (modelA predictions) to our list, pred now has our entire validation set predictions\n",
    "pred.extend(new_valid_labels_zero)\n",
    "\n",
    "#Add our validation labels to the real labels modelA said was 0, this is now the original full list for the validation labels\n",
    "new_valid_labels.extend(new_full_labels_zero)\n",
    "\n",
    "#Work out the MSE\n",
    "error = mean_squared_error(new_valid_labels, pred)\n",
    "print(\"The MSE for step 3 is\",error)"
   ]
  },
  {
   "source": [
    "This reinforces what was seen in the validation sets and further confirms that the single model is more effective on the dataset."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Question 5 - Present your solution [4 marks]\n",
    "\n",
    "Provide four interesting and meaningful observations/comments about your machine learning pipeline, with minimum three sentences for each observation/comment. "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Observation 1 - A good grid search can be vital\n",
    "\n",
    "Whilst doing this assignment I started without a grid search in place on the models and I just used the default values. Whilst for linear regression this basically made no impact, the other models saw a dramatic increase in score once optimised, especially the trees. An optimised model increased the MSE by around 5K from 54K to 49K. This really suprised me how much tuning just a few hyper parameters on the model can help the score. If I had more time (or a GPU set up) and could try optimise even more this could lead to an even greater model in place."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Observation 2 - Data can be messy and hard to process\n",
    "\n",
    "The data provided to us in the assignment had a lot of issues to be dealt with, mainly the NaN values in the categorys. This pre-processing was pivatol to the results and as such I spent a lot of time trying to do the best I could. I tried to use an advance imputation method with KNN and then I also under sampled the data. This has helped classification and made the data usable but these intial steps are important and a lot of focus went in there. If you don't have clean and well organised data the model can never be effective."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Observation 3 - You can't do much if the data is highly uncorrelated/independent of the label feature\n",
    "\n",
    "However, following on from above, it doesn't really matter what pre-processing you do if the data you end up with is very uncorrelated. The co-varience matrix I created for myself using a spearman ranking showed that the highest rated feature was only giving a correlation of 0.08. This is very very low and is really not showing a correlation at all. Ideally some of the features need to be showing some statisical signficance or the model will be innefective as seen in this data. The pre work done before is irrelevant if the data still doesn't capture the labels in a representative way."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Observation 4 - A more complicated model doesn't always help\n",
    "\n",
    "Whilst progressing through the assignment I expected the dual model in question 3 to be far more effective than the single models found in part 2. I was however proven wrong with this and it is in fact worse. Sometimes a simple solution can be just as effective if not better at predicting values. The data was not very reprsentative of our labels and by having 2 models we were forcing the model to predict things it was incapable of with the classes, increasing our error. It would be interesting to see if adding a feature indicating if it is 0 or not would help."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "Question 6 - Create a function that contains the best model you built from Steps 1 to 4 that we will use to assess the performance of your design over an independent test set [3 marks]."
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The best model so far is the GradientBoostingRegressor from Q2 part 4, so I will implement that as a function\n",
    "def my_insurance_claim_predictor(Xtest):\n",
    "    #Pre process the test data for our needs\n",
    "    Xtest = Xtest.replace('?', np.nan)\n",
    "\n",
    "    columns_to_remove = [\"Cat1\",\"Cat3\",\"Cat6\",\"Cat8\",\"Cat10\",\"Cat11\",\"Cat12\",\"OrdCat\",\"Blind_Make\",\"Blind_Model\",\"Blind_Submodel\"]\n",
    "    Xtest = Xtest.dropna(subset=columns_to_remove)\n",
    "    Xtest = Xtest.reset_index(drop=True)\n",
    "\n",
    "    cat_variables = claim_data[[\"Cat1\",\"Cat2\",\"Cat3\",\"Cat4\",\"Cat5\",\"Cat6\",\"Cat7\",\"Cat8\",\"Cat9\",\"Cat10\",\"Cat11\",\"Cat12\",\"OrdCat\",\"NVCat\"]]\n",
    "    cat_dummies = pd.get_dummies(cat_variables)\n",
    "\n",
    "    Xtest_ohe = pd.concat([Xtest.drop(cat_variables, axis=1), cat_dummies], axis=1)\n",
    "\n",
    "    scaler = MinMaxScaler()\n",
    "    #Too much data to onehotencode and package issues stopped me from hashing these - Not ideal but has to be done\n",
    "    #Also remove the target variable\n",
    "    Xtest_ohe = Xtest_ohe.drop([\"Blind_Make\",\"Blind_Model\", \"Blind_Submodel\", \"Claim_Amount\"], axis=1)\n",
    "\n",
    "    #Normalise the data to between 0 and 1\n",
    "    Xtest_ohe = pd.DataFrame(scaler.fit_transform(Xtest_ohe), columns= Xtest_ohe.columns)\n",
    "\n",
    "    #Find with KNN\n",
    "    imputer = KNNImputer(n_neighbors=7)\n",
    "\n",
    "    Xtest_imputed = pd.DataFrame(imputer.fit_transform(Xtest_ohe), columns= Xtest_ohe.columns)\n",
    "\n",
    "    #Reverse one hot encoding\n",
    "    for cat in columns_to_inv:\n",
    "        new_column = pd.DataFrame(Xtest_imputed.filter(regex=\"^\" + cat + \"_\").idxmax(axis=1), columns=[cat])\n",
    "        regex_string = cat + \"_\"\n",
    "        Xtest[cat] = new_column[cat].str.replace(regex_string,'')\n",
    "\n",
    "    drop_cols = [\"Row_ID\",\"Cat12\", \"Vehicle\", \"Cat10\", \"Cat11\", \"Blind_Make\", \"Model_Year\",\"Cat4\",\"Household_ID\", \"Calendar_Year\", \"Cat2\", \"Blind_Model\",\"Blind_Submodel\"]\n",
    "    test_data = Xtest.drop(drop_cols, axis = 1)\n",
    "\n",
    "    test_features = test_data.drop([\"Claim_Amount\"], axis=1)\n",
    "\n",
    "    #Transform the test features for tree use\n",
    "    test_features_transformed = tree_transformer.transform(test_features)\n",
    "\n",
    "    gbr = GradientBoostingRegressor()\n",
    "\n",
    "    gbr.set_params(**gbr_best)\n",
    "\n",
    "    gbr.fit(claim_data_train_under_features_transformed, claim_data_train_under_labels)\n",
    "\n",
    "    #Return my list of predictions\n",
    "    pred = gbr.predict(test_features_transformed)\n",
    "\n",
    "    return pred"
   ]
  },
  {
   "source": [
    "Thanks for reading the assignment! I hope it all made sense (even with some less than ideal variable names). :)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ]
}