Applied Machine Learning Cyber Threat Detection. ipynb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c8eea795",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('D:\\Birkbeck\\MSc Data Science\\Year 2\\AML\\Course work\\Coursework Dataset-20211201')\n",
    "filename = 'train_imperson_without4n7_balanced_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f2798633",
   "metadata": {},
   "outputs": [],
   "source": [
    "# All imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "\n",
    "from matplotlib import pyplot\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.pipeline import FeatureUnion, Pipeline\n",
    "from sklearn.model_selection import train_test_split, KFold, cross_val_score, RandomizedSearchCV, GridSearchCV\n",
    "from sklearn.feature_selection import VarianceThreshold, SelectKBest, chi2, mutual_info_classif\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "\n",
    "from sklearn.metrics import roc_curve, roc_auc_score, confusion_matrix, classification_report\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras.models import Model\n",
    "from keras.layers import Lambda, Input, Dense\n",
    "from keras.losses import binary_crossentropy\n",
    "\n",
    "from tensorflow.python.framework.ops import disable_eager_execution\n",
    "disable_eager_execution()\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab10734b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 97044 entries, 0 to 97043\n",
      "Columns: 152 entries, 1 to 155\n",
      "dtypes: float64(47), int64(105)\n",
      "memory usage: 112.5 MB\n"
     ]
    }
   ],
   "source": [
    "#Create DF exclude feature 38, see notes and [4]\n",
    "train = pd.read_csv(filename)\n",
    "train_data = train.drop(labels = '38', axis =1, inplace = False)\n",
    "train_data.info()\n",
    "# train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "180e3583",
   "metadata": {},
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2fa4a77c",
   "metadata": {},
   "outputs": [],
   "source": [
    "array = train_data.values\n",
    "X_train = array[ : , 0:151]\n",
    "y_train = array[ : , 151]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "56233f13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77635, 151)\n"
     ]
    }
   ],
   "source": [
    "#split into a train and validation set\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=3)\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cd002305",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "38851.0\n",
      "9671.0\n"
     ]
    }
   ],
   "source": [
    "#check that train and test still splits across the target evenly:\n",
    "print(y_train.sum())\n",
    "print(y_valid.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a138503",
   "metadata": {},
   "source": [
    "### Normalise Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "938438e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77635, 151)\n"
     ]
    }
   ],
   "source": [
    "norm_scaler = MinMaxScaler(feature_range = (0,1), clip= True).fit(X_train)\n",
    "n_X_train = norm_scaler.transform(X_train)\n",
    "print(n_X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a67627d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(19409, 151)\n"
     ]
    }
   ],
   "source": [
    "#also need to normalise the validation data for the autoencoder\n",
    "n_X_valid = norm_scaler.transform(X_valid)\n",
    "print(n_X_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de401083",
   "metadata": {},
   "source": [
    "# Variational Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ee276c22",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Attempt at Variational Autoencoder\n",
    "#coding layer is different in  Sampling class used as in Geron book [8] to calculate the mean and log variance in the code layer.\n",
    "\n",
    "class Sampling(keras.layers.Layer):\n",
    "    def call(self, input_data):\n",
    "        mean, log_var = input_data\n",
    "        return keras.backend.random_normal(tf.shape(log_var)) * keras.backend.exp(log_var / 2) + mean \n",
    "    \n",
    "input_size = 151\n",
    "code_size = 20\n",
    "\n",
    "input_data = Input(shape=(input_size,))\n",
    "hidden_1 = Dense(75, activation='relu')(input_data)\n",
    "hidden_2 = Dense(35, activation = 'relu')(hidden_1)\n",
    "\n",
    "code_mean = Dense(code_size)(hidden_2)\n",
    "code_log_var = Dense(code_size)(hidden_2)\n",
    "code = Sampling()([code_mean, code_log_var])\n",
    "\n",
    "var_encoder = Model(\n",
    "    input_data, outputs = code)\n",
    "# code_mean and code_log_variance not included in output, no need to inspect.\n",
    "\n",
    "decoder_inputs = Input(shape = [code_size])\n",
    "hidden_3 = Dense(35, activation = 'relu')(decoder_inputs)\n",
    "hidden_4 = Dense(75, activation='relu')(hidden_3)\n",
    "output_data = Dense(input_size, activation='sigmoid')(hidden_4)\n",
    "\n",
    "var_decoder = Model(inputs = decoder_inputs, outputs =output_data)\n",
    "\n",
    "code = var_encoder(input_data)\n",
    "reconst = var_decoder(code)\n",
    "var_ae = Model(input_data, reconst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b1eb7bab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 77635 samples, validate on 19409 samples\n",
      "Epoch 1/10\n",
      "77635/77635 [==============================] - 9s 113us/sample - loss: 0.0686 - val_loss: 0.0521\n",
      "Epoch 2/10\n",
      "77635/77635 [==============================] - 8s 100us/sample - loss: 0.0504 - val_loss: 0.0496\n",
      "Epoch 3/10\n",
      "77635/77635 [==============================] - 8s 99us/sample - loss: 0.0485 - val_loss: 0.0485\n",
      "Epoch 4/10\n",
      "77635/77635 [==============================] - 8s 99us/sample - loss: 0.0476 - val_loss: 0.0473\n",
      "Epoch 5/10\n",
      "77635/77635 [==============================] - 8s 98us/sample - loss: 0.0471 - val_loss: 0.0469\n",
      "Epoch 6/10\n",
      "77635/77635 [==============================] - 8s 99us/sample - loss: 0.0466 - val_loss: 0.0465\n",
      "Epoch 7/10\n",
      "77635/77635 [==============================] - 8s 97us/sample - loss: 0.0464 - val_loss: 0.0469\n",
      "Epoch 8/10\n",
      "77635/77635 [==============================] - 8s 100us/sample - loss: 0.0462 - val_loss: 0.0463\n",
      "Epoch 9/10\n",
      "77635/77635 [==============================] - 8s 98us/sample - loss: 0.0461 - val_loss: 0.0462\n",
      "Epoch 10/10\n",
      "77635/77635 [==============================] - 8s 97us/sample - loss: 0.0460 - val_loss: 0.0462\n"
     ]
    }
   ],
   "source": [
    "K= keras.backend\n",
    "#custom loss function for VAE\n",
    "\n",
    "def losses_func(input_data,reconst):\n",
    "    latent_loss = -0.5 * K.sum(1 + code_log_var - K.exp(code_log_var) - K.square(code_mean),axis = -1)\n",
    "    reconstruction_loss = tf.keras.losses.binary_crossentropy(input_data,reconst)\n",
    "    return (K.mean(latent_loss)/151.) + reconstruction_loss\n",
    "\n",
    "disable_eager_execution()\n",
    "var_ae.compile(optimizer = 'adam', loss = losses_func)\n",
    "history = var_ae.fit(n_X_train, n_X_train, epochs = 10, validation_data=[n_X_valid,n_X_valid])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "bb465ac8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAm90lEQVR4nO3deXRc5Z3m8e+vFu1LabMtW2VLBi/gRYVtjGl6CFvY0glJh06brE2nQ0gg3clMukNnZs4kp5ehM8kkoRtwQwKdTEiYxAkdJs0WAmTFgE2MbTDYxhhblhfZWIu1V9U7f9wrqywku2SVVJLq+ZxT51bdrX5VmHp03/fe95pzDhERyT2BbBcgIiLZoQAQEclRCgARkRylABARyVEKABGRHBXKdgGjUV1d7err67NdhojIlLJp06YjzrmaofOnVADU19ezcePGbJchIjKlmNmbw81XE5CISI5SAIiI5CgFgIhIjppSfQAiIqPV399PU1MTPT092S5l3BUUFFBXV0c4HE5rfQWAiExrTU1NlJaWUl9fj5llu5xx45zj6NGjNDU10dDQkNY2agISkWmtp6eHqqqqaf3jD2BmVFVVjepIRwEgItPedP/xHzDaz5kTAfD0q4e565ld2S5DRGRSyYkA+N3rR/jGkzvpiyezXYqI5JjW1lbuuuuuUW937bXX0tramvmCUuREADRGI/TFk7x2sCPbpYhIjhkpABKJxCm3e+SRR4hEIuNUlScnAiAWjQCwed+x7BYiIjnntttu4/XXXycWi3H++edz6aWX8sEPfpBly5YB8N73vpeVK1eyZMkS7rnnnhPb1dfXc+TIEfbs2cM555zDJz7xCZYsWcKVV15Jd3d3RmrLidNA50QKqS7JY/O+Nj5yYbarEZFs+fL/e5lXmtszus9zZ5fxP969ZMTlt99+O9u2bWPz5s0888wzvOtd72Lbtm0nTtW87777qKyspLu7m/PPP5/3v//9VFVVnbSPnTt38oMf/IB7772XD3zgA/z4xz/mwx/+8Jhrz4kAMDMa6yK81NSa7VJEJMetXr36pPP077jjDh566CEA9u3bx86dO98WAA0NDcRiMQBWrlzJnj17MlJLTgQAeM1AT712mPaefsoK0rtKTkSml1P9pT5RiouLTzx/5plnePLJJ3n22WcpKirikksuGfY8/vz8/BPPg8FgxpqAcqIPALyOYOdga1NbtksRkRxSWlpKR8fwJ6C0tbVRUVFBUVERr776Khs2bJjQ2nLmCKCxLgLA5n2tXHR2dXaLEZGcUVVVxUUXXcTSpUspLCxk5syZJ5ZdffXVrFu3juXLl7No0SLWrFkzobXlTACUF4WZX13M5n2t2S5FRHLM97///WHn5+fn8+ijjw67bKCdv7q6mm3btp2Y//nPfz5jdeVMExB4zUCb97XinMt2KSIiWZdTARCLRmjp6OVA2/QfFlZE5HRyKgAa/QvCXlIzkIhIbgXAObWl5AUD6gcQESHHAiA/FOSc2WUKABERciwAAGJ15Wzd30YiqY5gEcltuRcAcyN09SXYeVgjg4rI5FNSUgJAc3Mz119//bDrXHLJJWzcuHHM75VzATBwQZg6gkVkMps9ezbr168f1/fIuQCoryqmrCDE5n0aEkJExt8XvvCFk+4H8KUvfYkvf/nLXH755axYsYJly5bx05/+9G3b7dmzh6VLlwLQ3d3N2rVrWb58OX/6p3+q4aDPVCBgJy4IE5Ec8+htcHBrZvc5axlcc/uIi9euXctnP/tZPv3pTwPwwx/+kMcee4zPfe5zlJWVceTIEdasWcN73vOeEe/pe/fdd1NUVMSWLVvYsmULK1asyEjpOXcEAN4FYTsOddDVF892KSIyzZ133nkcPnyY5uZmXnrpJSoqKqitreWLX/wiy5cv54orrmD//v0cOnRoxH386le/OjH+//Lly1m+fHlGasu5IwDw+gESSce2/e2sbqjMdjkiMlFO8Zf6eLr++utZv349Bw8eZO3atTzwwAO0tLSwadMmwuEw9fX1ww4DnWqko4OxyMkjAF0RLCITae3atTz44IOsX7+e66+/nra2NmbMmEE4HObpp5/mzTffPOX2F198MQ888AAA27ZtY8uWLRmpKyePAGpK85kTKWSz7hAmIhNgyZIldHR0MGfOHGpra/nQhz7Eu9/9blatWkUsFmPx4sWn3P5Tn/oUN954I8uXLycWi7F69eqM1JWTAQDe9QCb97ZmuwwRyRFbtw52PldXV/Pss88Ou97x48cB76bwA8NAFxYW8uCDD2a8ppxsAgKI1UXY39pNS0dvtksREcmKnA2AgX6ALWoGEpEclbMBsHROGcGA6XoAkRyQKzeBGu3nzNkAKMoLsXBmqQJAZJorKCjg6NGj0z4EnHMcPXqUgoKCtLfJ2U5g8C4I+48tzSSTjkAg8+fYikj21dXV0dTUREtLS7ZLGXcFBQXU1dWlvX5aAWBmVwPfBILAt5xztw9Zbv7ya4Eu4M+ccy/6yyLAt4ClgAP+3Dn3rJl9CfgEMPBf5YvOuUfSrjwDYtFyfvD8XvYc7WR+TclEvrWITJBwOExDQ0O2y5iUTtsEZGZB4E7gGuBc4AYzO3fIatcAC/zHTcDdKcu+CTzmnFsMNALbU5Z93TkX8x8T+uMPKReEqSNYRHJQOn0Aq4Fdzrndzrk+4EHguiHrXAd813k2ABEzqzWzMuBi4NsAzrk+51xr5sofmwUzSinKC+p6ABHJSekEwBxgX8rrJn9eOuvMx2viud/Mfm9m3zKz4pT1bjWzLWZ2n5lVDPfmZnaTmW00s42ZbsMLBoxlc8rZ3KShoUUk96QTAMP1jg7tTh9pnRCwArjbOXce0Anc5i+/GzgLiAEHgK8N9+bOuXucc6ucc6tqamrSKHd0YtEI25vb6Y0nMr5vEZHJLJ0AaAKiKa/rgOY012kCmpxzz/nz1+MFAs65Q865hHMuCdyL19Q04WLRCH2JJNsP6BaRIpJb0gmAF4AFZtZgZnnAWuDhIes8DHzUPGuANufcAefcQWCfmS3y17sceAXAzGpTtn8fsG0sH+RMaWRQEclVpz0N1DkXN7NbgcfxTgO9zzn3spnd7C9fBzyCdwroLrzTQG9M2cVngAf88NidsuwrZhbDayraA3wyEx9otGrLC5hRms/mfa18LBsFiIhkSVrXAfinaD4yZN66lOcOuGWEbTcDq4aZ/5HRFDpezLxbROoIQERyTc4OBZEqFo2w+0gnbV392S5FRGTCKADwAgB0QZiI5BYFALCsrhwzdQSLSG5RAABlBWHOqinRyKAiklMUAL7GuggvNbVO+yFjRUQGKAB8sWg5R473sb+1O9uliIhMCAWALxb1hiJSM5CI5AoFgG/RrFLyQgF1BItIzlAA+PJCAZbMLtMRgIjkDAVAilg0wtb9bcQTyWyXIiIy7hQAKWLRCD39SXYcOp7tUkRExp0CIMXAFcFqBhKRXKAASDG3sohIUVgdwSKSExQAKczsxAVhIiLTnQJgiFg0wo5DHXT2xrNdiojIuFIADBGLRkg62LpfN4oXkelNATBEozqCRSRHKACGqCzOY25lkTqCRWTaUwAMQ7eIFJFcoAAYRiwaobmth8PtPdkuRURk3CgAhhGLlgPqBxCR6U0BMIwls8sJBUzXA4jItKYAGEZBOMji2lIdAYjItKYAGEFjXYQt+9pIJnWLSBGZnhQAI4hFI3T0xtl9RCODisj0pAAYweDIoLoiWESmJwXACObXlFCSH9L1ACIybSkARhAMGMvrytURLCLTlgLgFBqjEbYfaKenP5HtUkREMk4BcAqxaIR40vFyc3u2SxERyTgFwCkMdASrH0BEpiMFwCnMLCtgVlmBrggWkWlJAXAasWhEHcEiMi0pAE6jMRrhzaNdHOvsy3YpIiIZlVYAmNnVZvaame0ys9uGWW5mdoe/fIuZrUhZFjGz9Wb2qpltN7ML/fmVZvZzM9vpTysy97Eyp9EfGVTNQCIy3Zw2AMwsCNwJXAOcC9xgZucOWe0aYIH/uAm4O2XZN4HHnHOLgUZguz//NuAXzrkFwC/815PO8roIZhoaWkSmn3SOAFYDu5xzu51zfcCDwHVD1rkO+K7zbAAiZlZrZmXAxcC3AZxzfc651pRtvuM//w7w3jF9knFSkh9iwYwSnQkkItNOOgEwB9iX8rrJn5fOOvOBFuB+M/u9mX3LzIr9dWY65w4A+NMZZ1D/hBjoCHZOI4OKyPSRTgDYMPOG/hKOtE4IWAHc7Zw7D+hklE09ZnaTmW00s40tLS2j2TRjGqMRjnX1s++t7qy8v4jIeEgnAJqAaMrrOqA5zXWagCbn3HP+/PV4gQBwyMxqAfzp4eHe3Dl3j3NulXNuVU1NTRrlZl5jXQSAzeoIFpFpJJ0AeAFYYGYNZpYHrAUeHrLOw8BH/bOB1gBtzrkDzrmDwD4zW+SvdznwSso2H/Offwz46Vg+yHhaNKuUgnCAzXtbs12KiEjGhE63gnMubma3Ao8DQeA+59zLZnazv3wd8AhwLbAL6AJuTNnFZ4AH/PDYnbLsduCHZvZxYC/wJ5n5SJkXDgZYOrtcp4KKyLRy2gAAcM49gvcjnzpvXcpzB9wywrabgVXDzD+Kd0QwJcSiEf7PhjfpTyQJB3X9nIhMffolS1NjNEJvPMlrBzuyXYqISEYoANI0eIvI1qzWISKSKQqANNVVFFJVnKcAEJFpQwGQJjOjMRrRFcEiMm0oAEYhFo2wq+U4HT392S5FRGTMFACj0BiN4BxsbWrLdikiImOmABiFxjpvaGhdESwi04ECYBQiRXk0VBfrimARmRYUAKPUWKcrgkVkelAAjFJjNMKh9l4OtvVkuxQRkTFRAIzS4AVhx7JbiIjIGCkARumc2jLCQWPzPp0JJCJTmwJglArCQc6tLdMFYSIy5SkAzkBjNMKWplYSSd0iUkSmLgXAGWisi9DZl+D1luPZLkVE5IwpAM5AbG4EQNcDiMiUpgA4Aw1VxZQWhHRFsIhMaQqAMxAIGDGNDCoiU5wC4Aw11kV49WAH3X2JbJciInJGFABnqDEaIZF0vNys6wFEZGpSAJyhxqg/MqiagURkilIAnKEZpQXMiRQqAERkylIAjEFjVCODisjUpQAYg1g0wr63ujl6vDfbpYiIjJoCYAwa6yIAOgoQkSlJATAGy+rKCRgaGVREpiQFwBgU5YVYOLNUHcEiMiUpAMZo4Ipg5zQyqIhMLQqAMYpFI7R197PnaFe2SxERGRUFwBg1+reI1LhAIjLVKADGaOHMUoryguoHEJEpRwEwRsGAsXROuQJARKYcBUAGxKIRXmlupy+ezHYpIiJpUwBkQCwaoS+RZPuB9myXIiKSNgVABpzoCNYVwSIyhaQVAGZ2tZm9Zma7zOy2YZabmd3hL99iZitSlu0xs61mttnMNqbM/5KZ7ffnbzazazPzkSbe7PICqkvy1Q8gIlNK6HQrmFkQuBN4J9AEvGBmDzvnXklZ7Rpggf+4ALjbnw641Dl3ZJjdf90599UzLX6yMPNuEakAEJGpJJ0jgNXALufcbudcH/AgcN2Qda4Dvus8G4CImdVmuNZJLRYtZ3dLJ23d/dkuRUQkLekEwBxgX8rrJn9euus44Akz22RmNw3Z7la/yeg+M6sY7s3N7CYz22hmG1taWtIoNztiUa/8rU0aGE5EpoZ0AsCGmTd04JtTrXORc24FXjPRLWZ2sT//buAsIAYcAL423Js75+5xzq1yzq2qqalJo9zsWFY3cIvIY1muREQkPekEQBMQTXldBzSnu45zbmB6GHgIr0kJ59wh51zCOZcE7h2YP1WVF4aZX1OsoaFFZMpIJwBeABaYWYOZ5QFrgYeHrPMw8FH/bKA1QJtz7oCZFZtZKYCZFQNXAtv816l9BO8bmD+VDXQEa2RQEZkKTnsWkHMubma3Ao8DQeA+59zLZnazv3wd8AhwLbAL6AJu9DefCTxkZgPv9X3n3GP+sq+YWQyvqWgP8MkMfaasiUUj/OTF/TS39TAnUpjtckRETum0AQDgnHsE70c+dd66lOcOuGWY7XYDjSPs8yOjqnQKiKWMDKoAEJHJTlcCZ9DiWWXkBQO6HkBEpgQFQAblhQKcO7tMASAiU4ICIMNi0Qhbm9qIJzQyqIhMbgqADItFI3T3J9h5+Hi2SxEROSUFQIbpFpEiMlUoADKsvqqI8sKw+gFEZNJTAGSYmdGokUFFZApQAIyDWDTCjkMddPXFs12KiMiIFADjIBYtJ+k0MqiITG4KgHHQWBcBdItIEZncFADjoKokn2hlofoBRGRSUwCMk8a6CC9paGgRmcQUAOMkFo2wv7Wbwx092S5FRGRYCoBxMjgyqI4CRGRyUgCMkyWzywkGTFcEi8ikpQAYJ4V5QRbPKtWZQCIyaSkAxtHAFcHJpG4RKSKTjwJgHMXqInT0xHnjaGe2SxEReZvcCIBkErJwo/bY3AgAm/e2Tvh7i4icTm4EwPP3wAPXQ+veCX3bs2pKKM4Lqh9ARCal3AiAcAHs3QB3roEN6yCZmJC3DQaM5XURnQkkIpNSbgTAyj+DT2+AeX8Aj30B7rsKDm+fkLdujEZ45UA7Pf0TEzoiIunKjQAAiEThQz+CP74Xjr4O6/4TPP0/Id47rm8bi5bTn3BsP9A+ru8jIjJauRMAAGaw/ANw6wuw5H3wy9vhXy+Gfc+P21vGohUAGhhORCad3AqAAcXV8P574YM/gt7j8O0r4ZG/8Z5n2KzyAmaW5asfQEQmndwMgAELr4RbNsDqm7wzhe5aAzufzPjbxKIRXtLNYURkksntAADIL4VrvwJ//jiEi+CB98OPPwGdRzP2Fo3RCG8c6aS1qy9j+xQRGSsFwIC5F8DNv4Z33AYvPwR3ng9bfpSRC8hiJ+4QpqMAEZk8FACpQvlw6d/CJ38FFQ3wk7+A738AWveNabfL6soxQ/0AIjKpKACGM/Nc+PgTcPXtsOc3Xt/Ac/d4Q0qcgdKCMGfXlOhMIBGZVBQAIwkEYc2nvAvIoqvh0b+G+6+GltfOaHeNUe+KYJeFMYlERIajADidinnw4Z/A+/4VjuyAdX8Iz/wTxEfXoRuLRjja2UfTse5xKlREZHQUAOkwg8a1cMsLcM574Jl/hHveAU0b097FwC0i1QwkIpOFAmA0Smrg+m/DDf8XetrgW1fAo7eldQHZolml5IcC/F5DQ4vIJKEAOBOLrvb6Bs7/ODx3N9x1Iez6xSk3CQcDrG6o5P7fvcFnfvB7dhzqmKBiRUSGl1YAmNnVZvaame0ys9uGWW5mdoe/fIuZrUhZtsfMtprZZjPbmDK/0sx+bmY7/WlFZj7SBCkog3d9DW58zDt99Ht/DA/dDF1vjbjJHWvP4+Z3nMVT2w9x5dd/xae+t4lXmjVInIhkh53urBQzCwI7gHcCTcALwA3OuVdS1rkW+AxwLXAB8E3n3AX+sj3AKufckSH7/QrwlnPudj9UKpxzXzhVLatWrXIbN6bf7j5h+nvg11+F33wdCiJwzT/B0vd7fQfDONbZx/2/fYP7f7uHjt447zx3Jn952QKW1ZVPbN0ikhPMbJNzbtXQ+ekcAawGdjnndjvn+oAHgeuGrHMd8F3n2QBEzKz2NPu9DviO//w7wHvTqGVyChfAZf8NbvolRObCjz8OP1gLbfuHXb2iOI//fOUifnPbZXzuioU8t/so7/6X33Dj/c/z4t5jE1y8iOSqdAJgDpB6KWyTPy/ddRzwhJltMrObUtaZ6Zw7AOBPZwz35mZ2k5ltNLONLS0taZSbRbOWwl88CVf9I7zxK7jzAnj+3hEvICsvDPNXVyzgt7ddxl9ftYjN+1r547t+x0e+/Rwv7Bm5KUlEJBPSCYDh2jGGthudap2LnHMrgGuAW8zs4lHUh3PuHufcKufcqpqamtFsmh2BIFx4C3z6WahbCY98Hv7tWmjZMeImpQVhbrn0bH7zhcv422sWs/1AO3+y7lluuGcDz75+VBePici4SCcAmoBoyus6oDnddZxzA9PDwEN4TUoAhwaaifzp4dEWP6lV1MNH/h2uu8u7/eS6i+CX/+uUF5AV54f45DvO4td/cxn//Y/OZVfLcW64dwMf+Ndn+fXOFgWBiGRUOgHwArDAzBrMLA9YCzw8ZJ2HgY/6ZwOtAdqccwfMrNjMSgHMrBi4EtiWss3H/OcfA346xs8y+ZjBeR/y7kC2+F3w9N/D15fAv98Cr/wUeoY/A6gwL8jH/7CBX//NpXz5PUtoOtbNR779PO+763c8/ephBYGIZMRpzwKCE2f5fAMIAvc55/7BzG4GcM6tMzMD/gW4GugCbnTObTSz+Xh/9QOEgO875/7B32cV8ENgLrAX+BPn3CkbviftWUDp2vlz2Px9eP0X3oVkgbB3o/qFV8GCq6D67GE3640nWL+pibuefp39rd0sm1POZy47m3eeOxMb4UwjEZEBI50FlFYATBZTPgAGJOKw7znY+TjseAJatnvzK+d7QbDwKph3EYTyTtqsP5HkoRf38y9P72LvW12cU1vGX152NlctmUUgoCAQkeEpACazY2/Czidgx+Pe2UOJXsgrgfmX+EcHV0LprBOrxxNJfrq5mTuf3sXuI50snFnCrZct4F3LagkqCERkCAXAVNHX5YXAjse8UGj3ryWobfSPDq6G2edBIEAi6fjZlmb++ald7Dp8nPk1xXzmsrN59/LZhIIa5UNEPAqAqcg5OPTyYFNR0/PgklBcA2e/07up/VmXkcwr49FtB/nnp3by6sEO5lUVcculZ/O+8+YQVhCI5DwFwHTQ9RbsetJrKtr1JPS0QiAEcy+EhVeRPPtKfn64jDue2sXLze3UVRTy6UvO5vqVdeSFFAQiuUoBMN0k4tD0wmBT0WF/aKaKetyCK3mp8AL+/uUqNjZ1Mbu8gJsvOYsPrIpSEA5mt24RmXAKgOmuda/fkfwEvPFLiPfgwsUcnXkhP2o7l39rWYgrreXmd5zFDavnUpinIBDJFQqAXNLXBXt+7TUV7XwC2rxhmt4IncXPepbxYv751C+/mJUN1ayaV8ms8oIsFywi40kBkKuc84ai2Pk47Hgct+85zCV5y5XyTLKRpxMxdpVdwOKGuaycV8H59ZUsmFGi6wpEphEFgHi63oLXnyL52mMkdz5JqPcYCQK8ZIt4oq+Rp5LncTC/npXzKllVX8nKeRXEohH1HYhMYQoAebtkAvZv8o4Mdj6OHdwKwLHwTH7DeTzUuZTfJZeQCBawZHY559dXsKq+klXzKqgqyc9y8SKSLgWAnF5782BH8u5noL+TRDCfPaUreCYR44Fj57A7Xg1AQ3Uxq+ZVsMoPhfnVxRqXSGSSUgDI6MR74c3femGw83F4azcA3eVns6PsQp7oj/HDQ7Np6fb+/VQW57FyXoUfCpUsnVNGfkjNRiKTgQJAxubILu/oYOfjsOe3kOzH5ZdxvO5iXim+gEd7l/HL/cYbRzoByAsFiNVFWFlfwfn1FaycW0l5UTjLH0IkNykAJHN6O7wmoh2Pe0NcHz/ozZ+9gs55l/NS4QU81VbLxr1tbNvfRjzp/RtbOLOElfMqvb6EeZVEKwvVbCQyARQAMj6cg4NbBpuKmjYCDopnwIJ30ttwBVvyz+P5Awle2PMWm948RkdPHICa0nwWzyqlvqqYhupiGmqKmV9dzJxIoQazE8kgBYBMjM4jsOsXXhjsetK/8c3J4xXtSMxi45utvLj3GK+3dLK75fiJUAAIB425lUU0VJcwv8YPh2ovHGpK83XUIDJKCgCZeIm4N4LpQFPR4Ze9+RX13j0OFlwJ1QtxJTN5q9frP9h9pJM3jnTyRos/PdpJXzx5YpfFeUEaaoppqC45EQoN1cXUVxdTXqg+BpHhKAAk+1r3+R3JT8DuX0K8e3BZURWUzoayWiithbLZUFpLsqSWFqtgd28ZOzvy2H2kywuGI500HesimfLPt7ok78TRwomAqClmbmWRLmSTnKYAkMmlvxv2PQ+tb0L7AejwH+3N3rSz5e3bBPO9O6P54RAvmUlrsJrmRAVv9JXxWlcpL7UVsuNoPy0dvSc2M4M5kcKUcBhoUiphTkWh7qIm095IARDKRjEihAth/jtGXh7v884uGi4c2g/Agc2E2g9QHe+mGlieum1hJYnoLLryZ/BWsIoDyQr29Jezo62ErXuL+I/eMt6iFEeAvGCAaGUhsyOFzCwroLa84MR0VnkBs8oKqCzOU7+DTEsKAJmcQnkQmes9RuKc18k8NBw6mgl2HKS0vZnS1u3MO36YNfhHugYUQDIQpju/hmPBag4lK2g+Us7e5jJ295TwoqvgsKvgkIvQTjF5oSAzy/KpLStkZvnJITEwrSnNn1p3X3MOuo9Bx0EvaDsODU4tADWLYMY53rSgPNvVyjhRAMjUZQaFEe8x45yR10v0w/FDJx1NBNqbKe44QHF7M3Ud+1nZ8SIkOmBIP3I8kEdHuJpjrpLDrRH2HylnT28pryQiPEOEQ66Cwy5Cm5VQUzJ41DCrfMhzf1qUN87/yyUT3plYxw96P+4dB73PPnR6/BAk+t6+fV4pJOMn98+UzYGaxX4gLB4MhvzS8f0sMu4UADL9BcNQXuc9TqX3+OCPZMcBOH6IUMcBKjoOUXH8IPM7DkLHVgi2wZA+5biFaaeKo+2VHGwtpylexr7+cp49ERLeEUWyoJJZ5YVvD4ryAmpK8ikvDFNWEKakIHRy30S8b/CHe7i/2gemnYe9+0YPVVgBJbO8PpTqBVAy03s+dJpXDMmk1zfT8qo3lPjA9IXfQrxncJ/lUT8QFkPNOd60ehHkl5z5fyuZUOoEFhmtvq6T/8I+8YM85HVP29s2jVuItmAlR6igORGhKV7GoWSEQ1TQ5QqosVZm+I/aQBszA63U8BblruNt+3IYPXmV9BbOIFE8A1c8CyubRai8loKKWvIis7GBH/dQBkZvTSbg2J4hwfAqHNkBicFOdyJzBwOhxj9aqFnkhYtkhc4CEplo/d0pTS8HvL/Q/SOLgdfu+EGs+9hJmyUtxPFwFR2hKo4FKzlqXjPTwWQ5++Pl7O0rZU9vKYeSZSSGHoqkCAaMsoIQpQVhygpDlOaHKS0IUVboTwsGp2WF/noFYYrzgxTlhSjMC1KUFzx930Yi7gfDdi8QBqZHd6Y0M5kXDCc1Iy32giFcOLbvWU5LASAyWfX3eKHQ1wklM6CwEgKn/tF1ztHdn6CjJ05HTz9t3d603X/d7r/u6InTPjDtPvn18d74Kd9jQDhoFIaDfiCEKAx7wTAQEN6yEEV5KfPDQYpDUNW/n+qu3USOv05px06KWneR17YbS/Z7nwPDKurfHgzVCyGsW5Vmik4DFZmswgVQMW9Um5gZRXkhivJCzCw7sx/KRNJx3A+EgVDwgqGfrr4E3f6jq9+f9sVPzO/q88LncHsvXf3xE/O6+xO8/W/KKv+xGoAQcertIAutiYWBJha/tZ+Fx7Yw97XHCZEAIEmA7kARjgBYAGcBsKB3hlIggFkAAkEsEMQsiAWDWCBAwJ8XCAYJmPccf10sMOT54P5Ofh08eRnmnXBgaTwHf1/mz0vnOente9G1o/53cjoKAJEcFQwY5UXhjA7T7Zyjpz9Jd78XGAPB4IVDnO6+pDe/f3D+pr44v+5L0NfXQ+nxN6nqeoMZPW+Qn2gnmUiQSCRIJhK4ZAKSSYKWxHAESRI48fBeD85PEKCfcMARNkco4AjZyY+gOULmbRsyR4Ckv60jYEkCztu3d5ziwIGRBOe/xg0+d/56bnD+iam/bHC7M1S1QAEgIpOXmVHoNwNVFuedwR7OP+XSvnjSPyoZPBrp7I2nHKUMHql09SXoOmlZyvy+BN19cTr7BpclJ6g13MwRMggFIGReEAfNCyrvOYQDEPDXCQcgGID/GljK29pwxkgBICJTRl4oQF4oQPnQCzbGyDlHbzx5IkAGwqSzL04i6YgnHcmkI5F0JJ0jkYSEG5x34vlJ8zgxL5EcfCTdkG2SDDNv8Hnc36awMPOd5QoAEcl5ZkZBOEhB+EyPXKamKXTtuoiIZJICQEQkRykARERylAJARCRHpRUAZna1mb1mZrvM7LZhlpuZ3eEv32JmK4YsD5rZ783sZynzvmRm+81ss/+4duwfR0RE0nXas4DMLAjcCbwTaAJeMLOHnXOvpKx2DbDAf1wA3O1PB/wVsB0oG7L7rzvnvnrm5YuIyJlK5whgNbDLObfbOdcHPAhcN2Sd64DvOs8GIGJmtQBmVge8C/hWBusWEZExSicA5gD7Ul43+fPSXecbwN8AwwxSzq1+k9F9ZlYx3Jub2U1mttHMNra0DHOfWBEROSPpXAg23M1Qh140Pew6ZvZHwGHn3CYzu2TI8ruBv/P39XfA14A/f9tOnLsHuAfAzFrM7M00ah5ONXDkDLedjvR9DNJ3cTJ9HyebDt/HsIMIpRMATUA05XUd0JzmOtcD7/E7eAuAMjP7nnPuw865QwMrm9m9wM84DedcTRr1DsvMNg43HGqu0vcxSN/FyfR9nGw6fx/pNAG9ACwwswYzywPWAg8PWedh4KP+2UBrgDbn3AHn3N865+qcc/X+dk855z4MMNBH4HsfsG2sH0ZERNJ32iMA51zczG4FHse7E+p9zrmXzexmf/k64BHgWmAX0AXcmMZ7f8XMYnhNQHuAT57JBxARkTMzpe4INhZmdpPfnyDo+0il7+Jk+j5ONp2/j5wJABEROZmGghARyVEKABGRHJUTAXC6sYxyhZlFzexpM9tuZi+b2V9lu6bJYLixqnKVmUXMbL2Zver/O7kw2zVli5l9zv//ZJuZ/cDMCrJdU6ZN+wBIGcvoGuBc4AYzOze7VWVNHPgvzrlzgDXALTn8XaQaGKtK4JvAY865xUAjOfq9mNkc4C+BVc65pXhnQK7NblWZN+0DgPTGMsoJ/rUZL/rPO/D+5x46rEdO0VhVg8ysDLgY+DaAc67POdea1aKyKwQUmlkIKOLtF8BOebkQAOmMZZRzzKweOA94LsulZNs3GHmsqlwzH2gB7vebxL5lZsXZLiobnHP7ga8Ce4EDeBe3PpHdqjIvFwIgnbGMcoqZlQA/Bj7rnGvPdj3ZkjpWVbZrmSRCwArgbufceUAnkJN9Zv7glNcBDcBsoNjMPpzdqjIvFwIgnbGMcoaZhfF+/B9wzv0k2/Vk2UV4Y1XtwWsavMzMvpfdkrKqCWhyzg0cFa7HC4RcdAXwhnOuxTnXD/wE+IMs15RxuRAA6YxllBPMzPDad7c75/53tuvJtlONVZWLnHMHgX1mtsifdTnwyik2mc72AmvMrMj//+ZypmGHeDqjgU5pI41llOWysuUi4CPAVjPb7M/7onPukeyVJJPMZ4AH/D+WdpPeuF7TjnPuOTNbD7yId/bc7/GHpZ9ONBSEiEiOyoUmIBERGYYCQEQkRykARERylAJARCRHKQBERHKUAkBEJEcpAEREctT/Byi2FBdg1b0PAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pyplot.plot(history.history['loss'],label = 'train')\n",
    "pyplot.plot(history.history['val_loss'], label = 'valid')\n",
    "pyplot.legend()\n",
    "pyplot.show()         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0ee49d0a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77635, 20)\n"
     ]
    }
   ],
   "source": [
    "#The trained encoder contains the reduced features from the code layer.\n",
    "encoded = var_encoder\n",
    "gen_feat = encoded.predict(n_X_train)\n",
    "print(gen_feat.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "799d1716",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "      <td>77635.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.007975</td>\n",
       "      <td>-0.014686</td>\n",
       "      <td>-0.007167</td>\n",
       "      <td>-0.011954</td>\n",
       "      <td>-0.002431</td>\n",
       "      <td>0.004256</td>\n",
       "      <td>-0.002721</td>\n",
       "      <td>-0.005618</td>\n",
       "      <td>0.009899</td>\n",
       "      <td>-0.014758</td>\n",
       "      <td>0.040330</td>\n",
       "      <td>-0.026863</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.013149</td>\n",
       "      <td>-0.015940</td>\n",
       "      <td>0.001999</td>\n",
       "      <td>0.002420</td>\n",
       "      <td>0.008556</td>\n",
       "      <td>-0.000616</td>\n",
       "      <td>0.002857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.996512</td>\n",
       "      <td>0.998149</td>\n",
       "      <td>0.992776</td>\n",
       "      <td>1.025150</td>\n",
       "      <td>0.998242</td>\n",
       "      <td>0.993627</td>\n",
       "      <td>0.994100</td>\n",
       "      <td>1.001477</td>\n",
       "      <td>0.990939</td>\n",
       "      <td>1.003381</td>\n",
       "      <td>1.002986</td>\n",
       "      <td>1.039674</td>\n",
       "      <td>1.004941</td>\n",
       "      <td>1.004675</td>\n",
       "      <td>0.999484</td>\n",
       "      <td>0.992996</td>\n",
       "      <td>1.003375</td>\n",
       "      <td>1.001045</td>\n",
       "      <td>0.995505</td>\n",
       "      <td>1.001547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.913124</td>\n",
       "      <td>-4.172613</td>\n",
       "      <td>-4.691636</td>\n",
       "      <td>-2.868741</td>\n",
       "      <td>-4.312706</td>\n",
       "      <td>-4.170174</td>\n",
       "      <td>-4.359895</td>\n",
       "      <td>-5.059746</td>\n",
       "      <td>-4.051183</td>\n",
       "      <td>-4.635496</td>\n",
       "      <td>-5.310431</td>\n",
       "      <td>-3.234968</td>\n",
       "      <td>-4.424361</td>\n",
       "      <td>-4.138099</td>\n",
       "      <td>-4.494929</td>\n",
       "      <td>-3.915896</td>\n",
       "      <td>-3.960107</td>\n",
       "      <td>-4.236534</td>\n",
       "      <td>-4.306322</td>\n",
       "      <td>-4.304927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.664418</td>\n",
       "      <td>-0.683975</td>\n",
       "      <td>-0.676355</td>\n",
       "      <td>-0.823501</td>\n",
       "      <td>-0.675304</td>\n",
       "      <td>-0.666235</td>\n",
       "      <td>-0.670050</td>\n",
       "      <td>-0.682302</td>\n",
       "      <td>-0.657388</td>\n",
       "      <td>-0.692571</td>\n",
       "      <td>-0.645140</td>\n",
       "      <td>-0.828727</td>\n",
       "      <td>-0.676868</td>\n",
       "      <td>-0.666965</td>\n",
       "      <td>-0.690321</td>\n",
       "      <td>-0.664430</td>\n",
       "      <td>-0.679163</td>\n",
       "      <td>-0.671059</td>\n",
       "      <td>-0.670354</td>\n",
       "      <td>-0.671424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.010265</td>\n",
       "      <td>-0.010778</td>\n",
       "      <td>-0.005253</td>\n",
       "      <td>-0.035079</td>\n",
       "      <td>0.000632</td>\n",
       "      <td>0.005986</td>\n",
       "      <td>-0.001896</td>\n",
       "      <td>-0.005344</td>\n",
       "      <td>0.009249</td>\n",
       "      <td>-0.017381</td>\n",
       "      <td>0.007542</td>\n",
       "      <td>-0.258150</td>\n",
       "      <td>0.005092</td>\n",
       "      <td>0.013097</td>\n",
       "      <td>-0.011374</td>\n",
       "      <td>0.002759</td>\n",
       "      <td>0.000395</td>\n",
       "      <td>0.002336</td>\n",
       "      <td>-0.003913</td>\n",
       "      <td>0.003248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.679081</td>\n",
       "      <td>0.655875</td>\n",
       "      <td>0.661702</td>\n",
       "      <td>0.639562</td>\n",
       "      <td>0.667362</td>\n",
       "      <td>0.675464</td>\n",
       "      <td>0.665887</td>\n",
       "      <td>0.671623</td>\n",
       "      <td>0.677310</td>\n",
       "      <td>0.661387</td>\n",
       "      <td>0.745989</td>\n",
       "      <td>0.868696</td>\n",
       "      <td>0.685431</td>\n",
       "      <td>0.687567</td>\n",
       "      <td>0.660342</td>\n",
       "      <td>0.677670</td>\n",
       "      <td>0.687836</td>\n",
       "      <td>0.683942</td>\n",
       "      <td>0.671120</td>\n",
       "      <td>0.678753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.366189</td>\n",
       "      <td>4.441914</td>\n",
       "      <td>4.120115</td>\n",
       "      <td>2.695797</td>\n",
       "      <td>4.643752</td>\n",
       "      <td>4.973356</td>\n",
       "      <td>4.380567</td>\n",
       "      <td>4.174656</td>\n",
       "      <td>4.328950</td>\n",
       "      <td>4.009584</td>\n",
       "      <td>4.438735</td>\n",
       "      <td>3.465535</td>\n",
       "      <td>5.702774</td>\n",
       "      <td>4.739316</td>\n",
       "      <td>4.011050</td>\n",
       "      <td>4.237863</td>\n",
       "      <td>4.123657</td>\n",
       "      <td>4.106179</td>\n",
       "      <td>4.307766</td>\n",
       "      <td>4.159869</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0             1             2             3             4   \\\n",
       "count  77635.000000  77635.000000  77635.000000  77635.000000  77635.000000   \n",
       "mean       0.007975     -0.014686     -0.007167     -0.011954     -0.002431   \n",
       "std        0.996512      0.998149      0.992776      1.025150      0.998242   \n",
       "min       -3.913124     -4.172613     -4.691636     -2.868741     -4.312706   \n",
       "25%       -0.664418     -0.683975     -0.676355     -0.823501     -0.675304   \n",
       "50%        0.010265     -0.010778     -0.005253     -0.035079      0.000632   \n",
       "75%        0.679081      0.655875      0.661702      0.639562      0.667362   \n",
       "max        4.366189      4.441914      4.120115      2.695797      4.643752   \n",
       "\n",
       "                 5             6             7             8             9   \\\n",
       "count  77635.000000  77635.000000  77635.000000  77635.000000  77635.000000   \n",
       "mean       0.004256     -0.002721     -0.005618      0.009899     -0.014758   \n",
       "std        0.993627      0.994100      1.001477      0.990939      1.003381   \n",
       "min       -4.170174     -4.359895     -5.059746     -4.051183     -4.635496   \n",
       "25%       -0.666235     -0.670050     -0.682302     -0.657388     -0.692571   \n",
       "50%        0.005986     -0.001896     -0.005344      0.009249     -0.017381   \n",
       "75%        0.675464      0.665887      0.671623      0.677310      0.661387   \n",
       "max        4.973356      4.380567      4.174656      4.328950      4.009584   \n",
       "\n",
       "                 10            11            12            13            14  \\\n",
       "count  77635.000000  77635.000000  77635.000000  77635.000000  77635.000000   \n",
       "mean       0.040330     -0.026863      0.004052      0.013149     -0.015940   \n",
       "std        1.002986      1.039674      1.004941      1.004675      0.999484   \n",
       "min       -5.310431     -3.234968     -4.424361     -4.138099     -4.494929   \n",
       "25%       -0.645140     -0.828727     -0.676868     -0.666965     -0.690321   \n",
       "50%        0.007542     -0.258150      0.005092      0.013097     -0.011374   \n",
       "75%        0.745989      0.868696      0.685431      0.687567      0.660342   \n",
       "max        4.438735      3.465535      5.702774      4.739316      4.011050   \n",
       "\n",
       "                 15            16            17            18            19  \n",
       "count  77635.000000  77635.000000  77635.000000  77635.000000  77635.000000  \n",
       "mean       0.001999      0.002420      0.008556     -0.000616      0.002857  \n",
       "std        0.992996      1.003375      1.001045      0.995505      1.001547  \n",
       "min       -3.915896     -3.960107     -4.236534     -4.306322     -4.304927  \n",
       "25%       -0.664430     -0.679163     -0.671059     -0.670354     -0.671424  \n",
       "50%        0.002759      0.000395      0.002336     -0.003913      0.003248  \n",
       "75%        0.677670      0.687836      0.683942      0.671120      0.678753  \n",
       "max        4.237863      4.123657      4.106179      4.307766      4.159869  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Having a look at the output further:\n",
    "gen_featdf = pd.DataFrame(gen_feat)\n",
    "gen_featdf.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "191e87f5",
   "metadata": {},
   "source": [
    "As expected from the VAE code layer, the generated features have a mean of zero and a standard deviation of 1. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0dabb66b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#normalise the generated features and create a dataframe with new column labels to easily identify vs original features\n",
    "\n",
    "norm_scaler20 = MinMaxScaler(feature_range = (0,1), clip = True).fit(gen_feat)\n",
    "n_gen_feat = norm_scaler20.transform(gen_feat)\n",
    "n_gen_feat = pd.DataFrame(n_gen_feat)\n",
    "new_col_headers = map(str,[160,161,162,163,164,165,166,167,168,169, 170,171,172,173,174,175,176,177,178,179])\n",
    "n_gen_feat.columns= new_col_headers\n",
    "#print(n_gen_feat.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4b7128b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77635, 171)\n"
     ]
    }
   ],
   "source": [
    "#add generated features dataframe to new features dataframe. With correct column/feature labels. \n",
    "\n",
    "num_data = open('col labels numeric.csv')\n",
    "num_lst = []\n",
    "for label in num_data:\n",
    "    label = label.rstrip()\n",
    "    num_lst.append(label)\n",
    "\n",
    "n_orig_feat = pd.DataFrame(n_X_train, columns = num_lst)\n",
    "#pd.set_option('display.float_format',  '{:,.4f}'.format)\n",
    "#print(n_orig_feat.head())\n",
    "#print(n_orig_feat.shape)\n",
    "\n",
    "n_all_feat = pd.concat([n_orig_feat,n_gen_feat],axis = 1)\n",
    "print(n_all_feat.shape)\n",
    "#pd.set_option('display.max_columns',161)\n",
    "#print(n_all_feat.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "51fb334a",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25ef7734",
   "metadata": {},
   "source": [
    "### Variance Threshold Filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "58b6e731",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['5', '6', '8', '9', '14', '15', '16', '18', '20', '26', '29', '43',\n",
      "       '47', '48', '50', '51', '52', '61', '62', '64', '66', '67', '68', '70',\n",
      "       '71', '72', '73', '75', '76', '77', '78', '79', '80', '82', '83', '84',\n",
      "       '86', '88', '89', '90', '93', '94', '97', '98', '104', '105', '106',\n",
      "       '107', '108', '109', '110', '111', '112', '113', '117', '118', '119',\n",
      "       '120', '121', '122', '123', '125', '126', '127', '128', '129', '130',\n",
      "       '133', '138', '140', '141', '142', '143', '144', '145', '146', '154',\n",
      "       '160', '161', '162', '163', '164', '165', '166', '167', '168', '169',\n",
      "       '170', '171', '172', '173', '174', '175', '176', '177', '178', '179'],\n",
      "      dtype='object')\n",
      "97\n"
     ]
    }
   ],
   "source": [
    "feat_sel = VarianceThreshold()\n",
    "feat_sel.fit_transform(n_all_feat)\n",
    "red_feat = pd.DataFrame(feat_sel.transform(n_all_feat))\n",
    "col_lst = n_all_feat.columns[feat_sel.get_support()]\n",
    "print(col_lst)\n",
    "print(len(col_lst))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6f9b7f3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(77635, 97)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "red_feat.columns = col_lst\n",
    "red_feat.shape\n",
    "#red_all_feat.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff1fc403",
   "metadata": {},
   "source": [
    "### Kbest with Chi2 Filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d61db0ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['8', '9', '47', '50', '51', '66', '67', '68', '70', '71', '73', '77',\n",
      "       '82', '94', '107', '110', '121', '122', '126', '127', '128', '129',\n",
      "       '130', '140', '143', '145', '146', '154', '163', '171'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "kbest = SelectKBest(score_func = chi2, k =30)\n",
    "kbest_out = kbest.fit_transform(red_feat, y_train)\n",
    "kbest_out = red_feat.columns[kbest.get_support()]\n",
    "print(kbest_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8b04b148",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(77635, 30)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "red_feat2 = pd.DataFrame(kbest.transform(red_feat))\n",
    "red_feat2.columns = kbest_out\n",
    "red_feat2.shape\n",
    "#red_feat2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b8640c6",
   "metadata": {},
   "source": [
    "### Kbest with MI Filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7f9d23fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['8', '9', '47', '50', '51', '66', '67', '68', '71', '77', '82', '140',\n",
      "       '154', '163', '171'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "kbest_mi = SelectKBest(score_func = mutual_info_classif, k = 15)\n",
    "kbest_mi_out = kbest_mi.fit_transform(red_feat2 , y_train)\n",
    "kbest_mi_out = red_feat2.columns[kbest_mi.get_support()]\n",
    "print(kbest_mi_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "10cb52ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(77635, 15)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_final = pd.DataFrame(kbest_mi.transform(red_feat2))\n",
    "X_train_final.columns = kbest_mi_out\n",
    "X_train_final.shape\n",
    "#X_train_final.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "13c0a6ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(77635, 13)\n"
     ]
    }
   ],
   "source": [
    "#checking - correlation function from [12] for future addition to process. Not used in this model\n",
    "\n",
    "def pairwise_corr(df, threshold = 0.9):\n",
    "    \n",
    "    matrix = df.corr()\n",
    "    mask = np.triu(np.ones_like(matrix,dtype = bool))\n",
    "    reduced_matrix = matrix.mask(mask)\n",
    "    col_to_drop = [col for col in reduced_matrix.columns if any (reduced_matrix[col] > threshold)]\n",
    "    return col_to_drop\n",
    "\n",
    "corr_columns = pairwise_corr(X_train_final, 0.95)\n",
    "X_train_corr = X_train_final.drop(corr_columns, axis = 1)\n",
    "print(X_train_corr.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8f710da8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['8', '9']\n"
     ]
    }
   ],
   "source": [
    "print(corr_columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e28cd1a",
   "metadata": {},
   "source": [
    "# Selecting Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aef7a696",
   "metadata": {},
   "source": [
    "### Initial Exploration of Possible Algorithms - no tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "57d59136",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR: 0.977214 (0.001372)\n",
      "LDA: 0.957622 (0.001666)\n",
      "QDA: 0.897976 (0.146554)\n",
      "KNN: 0.997785 (0.000426)\n",
      "ET: 0.999820 (0.000175)\n",
      "NB: 0.981168 (0.002130)\n",
      "LSVC: 0.978657 (0.001423)\n"
     ]
    }
   ],
   "source": [
    "#compare algs as in lab:\n",
    "array = X_train_final.values\n",
    "X = array[ : , 0:15]\n",
    "Y = y_train\n",
    "\n",
    "# prepare models\n",
    "models = []\n",
    "models.append(('LR', LogisticRegression(solver='liblinear')))\n",
    "models.append(('LDA', LinearDiscriminantAnalysis()))\n",
    "models.append(('QDA', QuadraticDiscriminantAnalysis()))\n",
    "models.append(('KNN', KNeighborsClassifier()))\n",
    "models.append(('ET', ExtraTreesClassifier(random_state = 3)))\n",
    "models.append(('NB', GaussianNB()))\n",
    "models.append(('LSVC', LinearSVC(random_state = 3)))\n",
    "\n",
    "# evaluate each model in turn\n",
    "results = []\n",
    "names = []\n",
    "scoring = 'accuracy'\n",
    "for name, model in models:\n",
    "  kfold = KFold(n_splits=10, random_state=3,shuffle=True)\n",
    "  cv_results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring)\n",
    "  results.append(cv_results)\n",
    "  names.append(name)\n",
    "  msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "  print(msg)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6e08a0c",
   "metadata": {},
   "source": [
    "**Notes on results:**\n",
    "\n",
    "LR - performs well and trains quickly. Assumes linearity. ***EXPLORE FURTHER/TUNE***\n",
    "\n",
    "LDA/QDA - not good enough. \n",
    "\n",
    "KNN performs well but takes too long to run. (requires the whole dataset to be stored?)\n",
    "\n",
    "Extra trees - performs too well, likely overfitting, wont peform well on test. Extra trees supposed to reduce overfitting. \n",
    "\n",
    "NB - simple and performs well. good candidate alg. ***EXPLORE FURTHER/TUNE***\n",
    "\n",
    "Linear SVC perfom well. In [4] a SVM with gradient descent and a linear function was used to reduce the training time. Time is good/quick here.Quicker than ET. ***EXPLORE FURTHER/TUNE***"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0df9c084",
   "metadata": {},
   "source": [
    "## Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3248ed48",
   "metadata": {},
   "source": [
    "### Logistic Regression Tuning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cde2e1a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=LogisticRegression(),\n",
       "                   param_distributions=[{'C': [0.01, 0.05, 0.08, 0.1, 0.3, 0.5,\n",
       "                                               0.8],\n",
       "                                         'solver': ['lbfgs', 'liblinear', 'sag',\n",
       "                                                    'saga', 'newton-cg']},\n",
       "                                        {'C': [0.01, 0.05, 0.08, 0.1, 0.3, 0.5,\n",
       "                                               0.8],\n",
       "                                         'penalty': ['l1'],\n",
       "                                         'solver': ['liblinear', 'saga']},\n",
       "                                        {'C': [0.01, 0.05, 0.08, 0.1, 0.3, 0.5,\n",
       "                                               0.8],\n",
       "                                         'penalty': ['none'],\n",
       "                                         'solver': ['lbfgs', 'sag', 'saga',\n",
       "                                                    'newton-cg']}],\n",
       "                   random_state=3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parameters available are: penalty, C, solver. \n",
    "# penalty and solver linked in that the penalty depends on which solver.\n",
    "\n",
    "param_grid = [\n",
    "    {'solver':['lbfgs','liblinear','sag','saga','newton-cg'],'C': [0.01,0.05,0.08,0.1,0.3,0.5,0.8]},\n",
    "    {'penalty': ['l1'],'solver':['liblinear','saga'],'C': [0.01,0.05,0.08,0.1,0.3,0.5,0.8]},\n",
    "    {'penalty': ['none'],'solver':['lbfgs','sag','saga','newton-cg'],'C': [0.01,0.05,0.08,0.1,0.3,0.5,0.8]},\n",
    "]\n",
    "\n",
    "lr_model = LogisticRegression()\n",
    "\n",
    "lr_search = RandomizedSearchCV(lr_model, param_grid ,random_state = 3, cv = 5)\n",
    "lr_search.fit(X_train_final, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d795a167",
   "metadata": {},
   "source": [
    "#### Best results for logistic regression:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a8090d65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'solver': 'newton-cg', 'penalty': 'none', 'C': 0.08}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "bc1518e0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=0.08, penalty='none', solver='newton-cg')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e6231a96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9790429574289947"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4bcd84a",
   "metadata": {},
   "source": [
    "This score is an improvement on the initial exploration of models above"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cacbb22a",
   "metadata": {},
   "source": [
    "### NB Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "101100a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=GaussianNB(),\n",
       "                   param_distributions=[{'var_smoothing': [1e-10, 1e-09, 1e-07,\n",
       "                                                           1e-05]}],\n",
       "                   random_state=3)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# smoothing default at 1e-9.\n",
    "\n",
    "param_grid3 = [\n",
    "    {'var_smoothing': [1e-10,1e-9,1e-7,1e-5]},\n",
    "    ]\n",
    "\n",
    "nb_model = GaussianNB()\n",
    "\n",
    "nb_search = RandomizedSearchCV(nb_model,param_grid3,random_state =3, cv=5)\n",
    "nb_search.fit(X_train_final, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "58bf56c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'var_smoothing': 1e-09}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "ff470247",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GaussianNB()"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "57f9f442",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9815160687834096"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70944620",
   "metadata": {},
   "source": [
    "A small improvement on the untuned model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb807e64",
   "metadata": {},
   "source": [
    "### LSVC Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "dcf92760",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=LinearSVC(random_state=3),\n",
       "                   param_distributions=[{'C': [0.1, 0.3, 0.5, 0.8, 1.0, 1.2,\n",
       "                                               1.5, 2.0],\n",
       "                                         'loss': ['sqaured_hinge', 'hinge'],\n",
       "                                         'penalty': ['l1', 'l2']}],\n",
       "                   random_state=3)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_grid3 = [\n",
    "    {'penalty': ['l1','l2'], 'loss': ['sqaured_hinge','hinge'], 'C':[0.1,0.3,0.5,0.8,1.0,1.2,1.5,2.0]},\n",
    "    ]\n",
    "\n",
    "lsvc_model = LinearSVC(random_state = 3)\n",
    "\n",
    "lsvc_search = RandomizedSearchCV(lsvc_model,param_grid3,random_state =3, cv=5)\n",
    "lsvc_search.fit(X_train_final, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "fe68793a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'penalty': 'l2', 'loss': 'hinge', 'C': 1.5}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "68df0776",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.5, loss='hinge', random_state=3)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7867772a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9841051072325626"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc_search.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3da974f2",
   "metadata": {},
   "source": [
    "A small improvement on the untuned model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d07b706",
   "metadata": {},
   "source": [
    "## 3 Models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "226f4b65",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR: 0.979030 (0.001293)\n",
      "NB: 0.981168 (0.002130)\n",
      "LSVC: 0.984363 (0.000897)\n"
     ]
    }
   ],
   "source": [
    "# Compare 3 models with tuned parameters.\n",
    "\n",
    "array = X_train_final.values\n",
    "X = array[ : , 0:15]\n",
    "Y = y_train\n",
    "\n",
    "# prepare models with best parameters output\n",
    "models3 = []\n",
    "models3.append(('LR', lr_search.best_estimator_))\n",
    "models3.append(('NB', nb_search.best_estimator_))\n",
    "models3.append(('LSVC', lsvc_search.best_estimator_))\n",
    "\n",
    "# evaluate each model in turn\n",
    "results = []\n",
    "names = []\n",
    "scoring = 'accuracy'\n",
    "for name, model in models3:\n",
    "  kfold = KFold(n_splits=10, random_state=3,shuffle=True)\n",
    "  cv_results = cross_val_score(model, X, Y, cv=kfold, scoring=scoring)\n",
    "  results.append(cv_results)\n",
    "  names.append(name)\n",
    "  msg = \"%s: %f (%f)\" % (name, cv_results.mean(), cv_results.std())\n",
    "  print(msg)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e43c820",
   "metadata": {},
   "source": [
    "# Reviewing the 3 Models with Various Evaluation Metrics on TEST DATA:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ecf67b5",
   "metadata": {},
   "source": [
    "### Transform Validation Set to Use with Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "94d0fe95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19409, 15)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1) Normalise data (done - using already normed validation data, normed at start for VAE)\n",
    "# n_X_valid = norm_scaler.transform(X_valid)\n",
    "#2) predict with encoder\n",
    "X_valid = encoded.predict(n_X_valid)\n",
    "\n",
    "#3) scale encoder output\n",
    "X_valid = norm_scaler20.transform(X_valid)\n",
    "\n",
    "#4) add 20 features from encoder output to normalised full dataset (remember need diff names or will over-write)\n",
    "X_valid_full = pd.DataFrame(n_X_valid)\n",
    "X_valid_gen = pd.DataFrame(X_valid)\n",
    "X_valid_df = pd.concat([X_valid_full,X_valid_gen],axis = 1)\n",
    "\n",
    "#5) feature selection 1 - variancethreshold\n",
    "X_valid = feat_sel.transform(X_valid_df)\n",
    "\n",
    "#6) feature selection 2 - Kbest with chi2\n",
    "X_valid = kbest.transform(X_valid)\n",
    "\n",
    "#7) feature selection 2 - Kbest with chi2\n",
    "X_valid = kbest_mi.transform(X_valid)\n",
    "\n",
    "X_valid.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "ed32e494",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19409,)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Validation target class is stored in, check dimensions:\n",
    "y_valid.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7aa13030",
   "metadata": {},
   "source": [
    "## Logistic Regression Further Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a1e3ee23",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create CV scorng function:\n",
    "def scoring_cv(measure, model, X = X, Y = Y):\n",
    "    kfold = KFold(n_splits = 10, random_state = 3, shuffle = True)\n",
    "    cv_results = cross_val_score(model, X, Y, cv=kfold, scoring=measure)\n",
    "    return measure, round(cv_results.mean(),5), round(cv_results.std(),5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "668777de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('accuracy', 0.97903, 0.00129)\n",
      "('neg_log_loss', -0.06185, 0.00379)\n",
      "('roc_auc', 0.9974, 0.0003)\n"
     ]
    }
   ],
   "source": [
    "print(scoring_cv('accuracy',lr_search.best_estimator_))\n",
    "print(scoring_cv('neg_log_loss',lr_search.best_estimator_))\n",
    "print(scoring_cv('roc_auc',lr_search.best_estimator_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "911a67d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.997\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjCElEQVR4nO3deXxU5b3H8c8vO4SQAGFPwhJ2RBTCIoooiiJVqa29blfqUhHrhtRWWuvS6u1tr3WtC8WluLSidaVKVdwAQWRRdgTCHkD2HbLNPPePCTaEkExgJpMz832/XjF5zjmZ+R2T1zcPz5z5HXPOISIi3hcX6QJERCQ0FOgiIlFCgS4iEiUU6CIiUUKBLiISJRIi9cSZmZmubdu2kXp6ERFPmjdv3nbnXNPK9kUs0Nu2bcvcuXMj9fQiIp5kZuuOtU9LLiIiUUKBLiISJRToIiJRQoEuIhIlFOgiIlGi2kA3sxfMbKuZLT7GfjOzJ8ws38wWmlmv0JcpIiLVCeayxQnAk8BLx9h/AdCx7KMf8EzZZwmFKffBskmQ0Rb2bgQzaHEyHNwO9TMDn7sOh7xrAse/eQPkT4FG7aF4f+D4fjf9Zz/A3Amw7N0jv6+q7SISOhtmw9rp0HYgZPcN6UNbMO1zzawt8J5z7qRK9v0V+Nw592rZeDlwlnNuc1WPmZeX5yJ2HfqU+2DGY0dus3hwAH6wOIhLAOcgIRmSUqFeBiSmwtZl4HyBffUbQbtBgVA9tCewLy7+P8f3uynw2IdDEmDW01WHbPmQ3rXm6DqPpXEuFO4JfO+x9qe1hH2bYeeq4LeLSMj4i/Zg3y3BcJCQAj+dVONQN7N5zrm8yvaF4o1FrYEN5cYFZduOCnQzGwmMBMjJyQnBUwfhD1lQvK/645zvyK99ZePi4sD37//u6O/ZvwUWvX709sPHv3f7f7at+vTIY967HWY+UXmYHj7e4quv+7CDOwIz8qr2p7UMfK7JdhEJiQPFpezdupUW+AMbfMWBmXoIZ+mhCHSrZFul037n3HhgPARm6CF47mO7Pz2sDx8SxwrT43Hu72DdjMr/wBzen3dN4F8C5f/QVLddRE5IYYmPxz9Zyfhpqzmz3hqei/898f4SiE8KLLuEUCgCvQDILjfOAjaF4HGPz5N9YfvyiD19jRwrTA8bcCvM/EvZvx4M0rMhqf6x19APB3BVa+iHP1dcKz/WdhE5ISNfnse0Fdv4Se8sfvuDIcTv6FOn19B/ANwCDCPwYugTzrlqqwzLGvqJzMrr4hp63jVhfQFFRMJjf1EpCXFGSmI8X67aQanfz8COlfbTqrGq1tCrDXQzexU4C8gEtgD3AYkAzrlxZmYEroIZChwErnXOVZvUIQ/04wnz9oNhxNuhq0FEYt7UFdv4zVuL+OGprfjl+V1C/vgn9KKoc+6KavY74ObjrC00ngxi5praHH65Ivy1iEhM2n2wmAfeW8abXxeQ2zSVwV2a1XoNEWufG1JVrZkryEUkzGbkb+f2ifPZfbCYW87uwC2DO5CSWIOr1ELE+4Fe1VLL/Xtqrw4RiVlNGiSR3bgeL17Xh+6tIneFnfcD/VgU5iISJs453phXwJJNe7n/4u50adGQt24aQOAlxcjxdqA/cIw1qvaDa7cOEYkZG3Ye5DdvL2L6yu30bduYwhIfKYnxEQ9z8Hqg+4oq364rV0QkxHx+x0tfruX/PlhOnMEDPzyJq/rmEBcX+SA/zLuBvmF25duT0mq3DhGJCTsPFPPIlBX0a9+Y/7mkB60z6kW6pKN4N9CfP6/y7b8pqN06RCRqlfj8vPPNRn7cK4umacm8f+tAshvXqxPLK5XxbqBX2i6mbv5PFhHvWVSwh1++sYBvv9tHs4YpDOrUlJwm9SNdVpU8HOiVuH93pCsQEY8rLPHx2McreXb6apqkJvHXq3szqFNo3rYfbtEV6CIiJ+iGl+YyfeV2Lu+Tza+HdSW9XmKkSwqaAl1EYt6+whIS4+NISYzn5rM7MGpQLqd3yIx0WTXmzZtEH+sKFxGRGvrs262c/+g0nvhkJQD92zfxZJiDV2fo742JdAUi4nE7DxTzwHtLefubjXRs1oBzuzWPdEknzJuBvmvt0dt0/bmIBGn6ym2MnjifPYdKuO2cjtx8di7JCbXfTCvUvBnoft/R23T9uYgEqVlaCu0yU3nwkpPo0qJhpMsJGW+uoZcejHQFIuIhzjkmzl7PPe8sBqBzizT+Oeq0qApz8OoMXUQkSOt3HGTsWwuZuWoH/dvXrWZaoaZAF5Go5PM7/jZjDX/+aDkJcXH84ZIeXN4nu0410wo1BbqIRKWdB4p5/JOVnJ6byYOXnETL9LrXTCvUPBrocYC/wlhEYl1xaaCZ1qW9A820Jt82kKxGdbeZVqh5M9DTs2DP+iPHIhLTFmzYza/eWMjyLftokZ7CmZ2akt24bjfTCjVvBvrejVWPRSRmHCr28ciU5Tz/xRqapaXw3Ig8zvRIM61Q82agO1/VYxGJGTe8NJcv8rdzRd8cfj2sCw1TvNNMK9S8GegpjaBw15FjEYkZewtLSCprpnXr4A78/OxcBuR6s/9KKHnz1cSKr2/ExusdIgJ8smwL5z0yjcfLmmn1a99EYV7Gm4HeqH3VYxGJOjv2F3Hbq99w/YtzyaifyNDuLSJdUp3jzSWX7SuqHotIVJm2YhujX5vPvsIS7ji3EzedlUtSgjfno+HkzUAv3lf1WESiSov0FDo0bcCDl5xEp+bqrHos+hMnInWO3+/4x1frufvtRQB0ap7G66NOU5hXw3sz9JcuOXpbqvcb04tIwNrtBxj71kJmrd7Jae2bfN9MS6rnvUBfP/Pobf1vrP06RCSkfH7HC1+s4eEpy0mMi+OPP+rBZX2yY+Zt+6EQ1JKLmQ01s+Vmlm9mYyvZn25m/zKzBWa2xMyuDX2pZXIGVHjyOGg7MGxPJyK1Y+eBYv7y6UrO6NCUKWMGcXnfHIV5DVUb6GYWDzwFXAB0A64ws24VDrsZWOqc6wmcBTxsZkkhrjWg2/AjxwNug+y+YXkqEQmvolIfr85ej9/vAs20bh/IsyN60yI9JdKleVIwM/S+QL5zbrVzrhiYCFRIVRyQZoE/pw2AnUBpSCs97Ktnjhyv+CAsTyMi4fXN+l1c9Jcv+PVbi/gifzsAWY3qa1Z+AoJZQ28NbCg3LgD6VTjmSWASsAlIAy5zzvkrHIOZjQRGAuTk5BxPvbBzTdVjEanTDhaX8vBHK3hhxhpaNEzhb9f0idlmWqEWTKBX9ufSVRifD8wHBgO5wBQzm+6c23vENzk3HhgPkJeXV/ExguMrqnosInXayJfm8UX+dv67fw53De1CWgw30wq1YAK9AMguN84iMBMv71rgj845B+Sb2RqgCzA7JFWWF598ZIjHJ4f8KUQktPYcKiE5IdBM67ZzOnLr4A70a98k0mVFnWDW0OcAHc2sXdkLnZcTWF4pbz1wDoCZNQc6A6tDWaiIeNOUpVs479GpPPZxoJlW33aNFeZhUu0M3TlXama3AB8C8cALzrklZjaqbP844AFggpktIrBEc5dzbntYKvaVVD0WkTph+/4i7p+0hPcWbqZLizSG9VAzrXAL6o1FzrnJwOQK28aV+3oTcF5oSzuGVqfCpnlHjkWkTvl8+VZGvzafg0U+fjGkE6POyiUxXp1Gws177xTtNeLIQO81InK1iEilWmXUo3PzNB784Ul0VP+VWuO9P5kVr0OvOBaRWuf3O16etY5fv/WfZlqv3XiawryWeW+Gvmdj1WMRqVWrt+1n7JuLmL12JwM7ZqqZVgR5L9D9vqrHIlIrSn1+np2+hkc/XkFKQhwPXXoyl/bO0js9I8h7gZ6aCXvWHzkWkVq362AJ46au4uzOTXlg+Ek0a6j+K5HmvUDX3YpEIqao1Mcb8wq4ok8OTdOS+fftA2mVUS/SZUkZ7wV6Uhoc2nXkWETCbt66Xdz15kLyt+6nTeNUzuiYqTCvY7wX6Ad3VD0WkZA6UFTKnz9azoSZa2mVXo8Xr+vLGR211FkXeS/QLa7qsYiE1MiX5zIjfwc/Pa0NvxzahQbJ3ouNWKGfjIgcZc/BEpITA820Rp/bidHnQp+2jSNdllTDe9NbXbYoElYfLN7MuY9O5dGPVwCBIFeYe4P3Ar1Z16rHInJctu4r5KZX5jHqla9p2iCZi05uFemSpIa8t+RSL73qsYjU2GfLtzJ64nwOlfj45fmdGXlmezXT8iDvBXrX4bDq0yPHInJCsjLq0b1VQ34//CQ6NGsQ6XLkOHnvT/C6GVWPRaRafr/jxZlrGfvmQgA6Nk/jHzf0V5h7nPdm6N++X/VYRKq0att+7npjIXPX7eLMTk3VTCuKeC/QkxpAyYEjxyJSrRKfn/HTVvP4JyuplxjPn3/Skx/3aq1mWlHEe0su7QdVPRaRSu05VML4aas5t2szpow5U50Ro5D3ZugbZh05/m5hZOoQ8YDCEh//nLuBq/q1IbNBMh+MHkjLdPVfiVbeCvQNs2H3+iO3FR+MTC0iddyctTu5642FrN5+gHaZDTijY6bCPMp5K9DXTj96W8setV+HSB22v6iU//vgW176ch1Zjerx8vVqphUrvBXobQcCBrjA2OLh9NERLEik7hn50ly+XL2Da09vy53ndSZVzbRihrd+0tl9oUUP2L8VugyDnlcEtonEuN0Hi0lOiKdeUjy/OK8TYPRu0yjSZUkt81agAyQ3DHxc+GikKxGpEyYv2sy97y7mx72y+PWwrvRuo0Zascp7gS4iAGzdW8g97y7mwyVb6NE6neGntI50SRJhCnQRD/r02y2MnjifolI/Yy/ows/OaEeCmmnFPAW6iAflNK5Pz+wMfndxd9o31bulJUB/0kU8wOd3vPDFGn71xgIAOjRL4+Xr+ynM5QiaoYvUcSu37OOuNxfy9frdnN1ZzbTk2BToInVUcamfv05dxV8+zSc1OZ7HLjuF4ae0Uv8VOaagllzMbKiZLTezfDMbe4xjzjKz+Wa2xMymhrZMkdizt7CE52es4bzuzZkyZhA/PFWdEaVq1c7QzSweeAoYAhQAc8xsknNuabljMoCngaHOufVm1ixM9YpEtcISH6/N2cDV/QPNtD4cfSbNG6ZEuizxiGBm6H2BfOfcaudcMTARqHjftyuBt5xz6wGcc1tDW2Y5+zbDlsUwd0LYnkIkEr5avYMLHp/OfZOW8OXqHQAKc6mRYAK9NbCh3LigbFt5nYBGZva5mc0zsxGVPZCZjTSzuWY2d9u2bTWvdu4E2LkKCnfDe7cr1CUq7Css4bfvLOKy8bMo9fv5+8/6cXoHNdOSmgvmRdHKFu1cJY/TGzgHqAd8aWaznHMrjvgm58YD4wHy8vIqPkb1lr179Djvmho/jEhdMvKlecxas4Prz2jHL87rRP0kXasgxyeY35wCILvcOAvYVMkx251zB4ADZjYN6AmsIJS6DodVnx45FvGgnQeKqZcYaKZ15/mdMYNeOWqmJScmmCWXOUBHM2tnZknA5cCkCse8Cww0swQzqw/0A5aFtlQCs/HGuZCSARc+rtm5eI5zjkkLNnHuI1N59OPAfKd3m0YKcwmJamfozrlSM7sF+BCIB15wzi0xs1Fl+8c555aZ2QfAQsAPPOecWxyWitNaBj4U5uIx3+0p5LfvLObjZVvomZXOj3qpmZaEVlCLdc65ycDkCtvGVRg/BDwUutJEoscnywLNtEr8fu4e1pXrzmhHfJyuKZfQ0qsvIrWgTZNUerVpxO8u7k7bzNRIlyNRSs25RMLA53c8N301v3j9cDOtBrx4XV+FuYSVZugiIbZiyz5+9cZC5m/YzeAuzdRMS2qNAl0kRIpL/Tzz+Sqe/GwlaSmJPH75KVzcU820pPYo0EVCZG9hCRNmrmFYj5bce2E3mjRIjnRJEmMU6CIn4FCxj1dnr+enA9p+30yrmfqvSIQo0EWO08xV2xn75iLW7zxI5xZpnN4hU2EuEaVAF6mhvYUl/O/kb3l19nraNKnPqzf057TcJpEuS0SBLlJTI1+ay+w1O7nxzPaMPrcT9ZJ0BYvUDQp0kSDs2F9E/aQE6iXF86uhXYg3o2d2RqTLEjmC3lgkUgXnHO/O33hEM61eOY0U5lInaYYucgyb9xzit28v5pNvt3JKdgaX9s6KdEkiVVKgi1RiytIt3PHafHx+xz0XduOaAW3VTEvqPAW6SCXaZaaS17YRv7/4JHKa1I90OSJB0Rq6CFDq8zN+2irGvDYfCDTTmnBtX4W5eIr3An3fZtiyWDeIlpBZtnkvP3pmJn+Y/C37ikopLPFFuiSR4+KtJZe5E2DnqsDX790e+Kw7F8lxKir18dRnq3j6s3wy6ify1JW9GNajhZppiWd5a4a+7N2qxyI1sL+wlFdmrePinq2YcscgfnByS4W5eJq3Ar3r8KrHItU4WFzKc9NX4/M7mpQ103rkslNolJoU6dJETpi3Aj3vGmicCykZcOHjWm6RGpmRv53zH5vGg+8v46vVOwBomqYWtxI9vLWGDpDWMvChMJcg7TlUwh/eX8ZrczfQLjOV10b2p197NdOS6OO9QBepoRtfnsuctbsYNSiX0ed21O3gJGop0CUqbdtXRGpyPPWTErhraBcS4uLokZUe6bJEwspba+gi1XDO8dbXBQx5dCqPTgk00zo1p5HCXGKCZugSNTbuPsTdby/i8+Xb6JWTwWV9siNdkkitUqBLVPhoyXfc8dp8HHD/Rd24+jQ105LYo0AXT3POYWbkNmtA//ZNuP/i7mQ3Vv8ViU1aQxdPKvX5eebzVdxR1kwrt2kDnr+mj8JcYppm6OI5Szft5VdvLmDxxr2c3705hSU+XYooggJdPKSwxMeTn+YzbuoqMuon8cxVvbigR8tIlyVSZyjQxTMOFJXyj9nrGX5Ka+65sCsZ9dV/RaS8oNbQzWyomS03s3wzG1vFcX3MzGdml4auRIllB4pKGT9t1ffNtKbccSYP/1dPhblIJaqdoZtZPPAUMAQoAOaY2STn3NJKjvsT8GE4CpXYM23FNn791iI27TnESa3TGZCbSZMGaqYlcizBzND7AvnOudXOuWJgIlBZ39pbgTeBrSGsT2LQ7oPF3PnPBYx4YTbJiXH888bTGJCbGemyROq8YNbQWwMbyo0LgH7lDzCz1sAlwGCgz7EeyMxGAiMBcnJyalqrxIiRL89j3rpd3Hx2LrcOVjMtkWAFE+iVvd3OVRg/BtzlnPNVdccX59x4YDxAXl5exceQGLZ1XyENkhOon5TAb4Z1JTHe6N5K/VdEaiKYQC8AyjfFyAI2VTgmD5hYFuaZwDAzK3XOvROKIiV6Oed4Y14BD76/jJ/0zuK3F3bjlOyMSJcl4knBBPocoKOZtQM2ApcDV5Y/wDnX7vDXZjYBeE9hLtXZsPMgv3l7EdNXbqdP20Zc0U/LcCInotpAd86VmtktBK5eiQdecM4tMbNRZfvHhblGiUIfLP6OMa/Px4DfD+/Of/drQ5yaaYmckKDeWOScmwxMrrCt0iB3zl1z4mVJtDrcTKtT8wac3iGT+y7qRlYj9V8RCQU155JaUeLz89Rn+dw+cT4A7Zs24NkReQpzkRBSoEvYLd64h+FPzuChD5fjc46iUl+kSxKJSurlImFTWOLj8U9WMn7aahqnJvHXq3tzfvcWkS5LJGop0CVsDhb7eH3OBn7cqzV3D+tGev3ESJckEtUU6BJS+4tKeWXWOm4Y2J7GqUlMGTOIxqlqpCVSGxToEjKfL9/K3W8vZtOeQ/TMyuC03CYKc5FapECXE7brQDEPvL+Ut77eSIdmDXhj1AB6t2kU6bJEYo4CXU7Yja/M4+t1u7htcAduHtyB5AQ10xKJBAW6HJetewtJTU4gNTmBu4d1JTE+jm6tGka6LJGYpuvQpUacc7w+ZwPnPDKVR6asAKBndobCXKQO0AxdgrZ+R6CZ1hf52+nbrjFXqZmWSJ2iQJegfLB4M3e8toD4OOPBH57ElX1z1ExLpI5RoEuVDjfT6tyiIYM6NeXei7rRKqNepMsSkUpoDV0qVVzq5y+frOS2ifNxztEuM5VxV/dWmIvUYQp0OcrCgt1c/OQXPFz2omexzx/hikQkGFpyke8Vlvh4dMoKnp2+mqZpyTw7Io8h3ZpHuiwRCZICXb53sNjHG/MKuKxPNmMv6Ep6PTXTEvESBXqM21dYwsuz1nHjmbk0Tk3i4zGDaKT+KyKepECPYZ9+u4W7317Mlr2FnJrdiNNymyjMRTxMgR6Dduwv4vfvLeXd+Zvo1LwBT181gFNz1ExLxOsU6DHople+5psNuxh9bkd+flYHkhJ0sZNINFCgx4jv9hSSlhJopnXPhd1ISoijc4u0SJclIiGkqVmUc87x6uz1DCnXTKtHVrrCXCQKaYYexdbtOMDYNxfx5eodnNa+CSNOaxPpkkQkjBToUWryos2MeX0+iXFx/O+PenB5n2zM1ExLJJop0KPM4WZaXVs2ZHCXZtxzYTdapqv/ikgs0Bp6lCgu9fPYxyu45dVvvm+m9fRVvRXmIjFEgR4F5m/YzUV/+YLHPl5JQpypmZZIjNKSi4cdKvbxyJTlPP/FGpqlpfD8T/M4p6uaaYnEKgW6hxWW+Hj7m01c0TeHsRd0IS1FzbREYllQSy5mNtTMlptZvpmNrWT/VWa2sOxjppn1DH2pArC3sIQnP11Jqc9Po9QkPhkziP+5pIfCXESqn6GbWTzwFDAEKADmmNkk59zScoetAQY553aZ2QXAeKBfOAqOZR8v3cLd7yxi274ierdpzGm5TUivryAXkYBgllz6AvnOudUAZjYRGA58H+jOuZnljp8FZIWyyFi3Y38R9/9rKf9asIkuLdJ4dkQeJ2dlRLosEaljggn01sCGcuMCqp59Xw/8u7IdZjYSGAmQk5MTZIlyuJnWmCGdGDUoV820RKRSwQR6ZW8vdJUeaHY2gUA/o7L9zrnxBJZjyMvLq/QxJGDznkM0TEkkNTmBey8KNNPq1Fz9V0Tk2IKZ6hUA2eXGWcCmigeZ2cnAc8Bw59yO0JQXe/x+x9+/WseQR6bx8EeBZlontU5XmItItYKZoc8BOppZO2AjcDlwZfkDzCwHeAu42jm3IuRVxog12w8w9s2FfLVmJ6d3aMI1A9pGuiQR8ZBqA905V2pmtwAfAvHAC865JWY2qmz/OOBeoAnwdFkDqFLnXF74yo4+7y8MNNNKSojj/358Mj/Jy1IzLRGpkaDeWOScmwxMrrBtXLmvfwb8LLSlxYbDzbS6t2rIkG7NuefCbjRvmBLpskTEg3S5RIQUlfp45KPl3PyPr3HO0TYzlSev7KUwF5HjpkCPgK/X7+LCJ77giU/zSUmIVzMtEQkJ9XKpRQeLS/nzhyv428w1tGyYwt+u7cPZnZtFuiwRiRIK9FpUVOLnXws3cXX/NvxqaBcaJOt/v4iEjhIlzPYcKuHFmWv5+Vm5NEpN4uMxg0ivp/4rIhJ6CvQw+nDJd9zzzmJ2HCimX7vG9GvfRGEuImGjQA+DbfuKuH/SEt5ftJmuLRvy/E/70CMrPdJliUiUU6CHwc//Po8FG/Zw53mduHFQLonxuphIRMJPgR4iG3cfIr1eIg2SE7jvou4kJ8TRUf1XRKQWaep4gvx+x0tfruW8R6bySLlmWgpzEaltmqGfgFXb9jP2zYXMWbuLgR0zufb0tpEuSURimAL9OL23cBNjXl9ASkIcD116Mpf2VjMtEYksBXoNHW6m1aN1OkO7t+C3F3alWZr6r4hI5GkNPUiFJT4e+vBbbnol0EyrTZNUnrjiVIW5iNQZCvQgzFu3kx88MZ2nPltFanKCmmmJSJ2kJZcqHCgq5aEPl/Pil2tplV6PF6/ry6BOTSNdlohIpRToVSjx+Zm8aDMj+rfhl2qmJSJ1nBKqgt0Hi/nbjLXcOrgDGfWT+PgXg2iYov4rIlL3KdDL+feizdzz7hJ2HSxmQG4T+rVvojAXEc9QoANb9xZy77tL+GDJd3Rv1ZAXr+tD91ZqpiUi3qJAB27+x9csKNjDXUO7cMPAdiSomZaIeFDMBnrBroNk1E+iQXIC91/cnZTEeHKbNoh0WSIixy3mpqJ+v2PCjDWc9+g0Hv5oOQDdW6UrzEXE82Jqhp6/NdBMa+66XQzq1JTrz2gX6ZJEREImZgJ90oJN3Pn6Auonx/PIf/XkklNbq5mWiESVqA90v98RF2f0zEpnWI8W3P2DbjRNS450WSIiIRe1a+iFJT7++O9vGfXKvO+baT12+akKcxGJWlEZ6LPX7GTY49MZN3UVjeonUeJzkS5JRCTsomrJZX9RKX/697e8PGsd2Y3r8cr1/TijY2akyxIRqRVRFeilPj8fLf2O605vx53nd6J+UlSdnohIlTyfeLsOFPO3GWu47ZyOZNRP4pNfnKWuiCISk4JaQzezoWa23MzyzWxsJfvNzJ4o27/QzHqFvtQjOed4f+Fmhjw6lac/X8XX63cDKMxFJGZVm35mFg88BQwBCoA5ZjbJObe03GEXAB3LPvoBz5R9Dr19m/Ed2M7fn/od9xb0oUfrdF66rh/dWjUMy9OJiHhFMNPZvkC+c241gJlNBIYD5QN9OPCSc84Bs8wsw8xaOuc2h7TauRNg5yrigKsLH6XNyXdz+mV3qpmWiAjBLbm0BjaUGxeUbavpMZjZSDOba2Zzt23bVtNaYdm7gccp+88g35cKcxGRMsGkYWXvj694YXcwx+CcG++cy3PO5TVtehz35uw6/MgnLDcWEYl1wSy5FADZ5cZZwKbjOObE5V0T+Lzs3UCYHx6LiEhQgT4H6Ghm7YCNwOXAlRWOmQTcUra+3g/YE/L188PyrlGQi4hUotpAd86VmtktwIdAPPCCc26JmY0q2z8OmAwMA/KBg8C14StZREQqE9RF2865yQRCu/y2ceW+dsDNoS1NRERqQpeIiIhECQW6iEiUUKCLiEQJBbqISJSwwOuZEXhis23AuuP89kxgewjL8QKdc2zQOceGEznnNs65St+ZGbFAPxFmNtc5lxfpOmqTzjk26JxjQ7jOWUsuIiJRQoEuIhIlvBro4yNdQATonGODzjk2hOWcPbmGLiIiR/PqDF1ERCpQoIuIRIk6Heh18ebU4RbEOV9Vdq4LzWymmfWMRJ2hVN05lzuuj5n5zOzS2qwvHII5ZzM7y8zmm9kSM5ta2zWGWhC/2+lm9i8zW1B2zp7u2mpmL5jZVjNbfIz9oc8v51yd/CDQqncV0B5IAhYA3SocMwz4N4EbGPUHvop03bVwzgOARmVfXxAL51zuuE8JdP28NNJ118LPOYPAfXtzysbNIl13LZzzb4A/lX3dFNgJJEW69hM45zOBXsDiY+wPeX7V5Rn69zends4VA4dvTl3e9zends7NAjLMrGVtFxpC1Z6zc26mc25X2XAWgbtDeVkwP2eAW4E3ga21WVyYBHPOVwJvOefWAzjnvH7ewZyzA9LMzIAGBAK9tHbLDB3n3DQC53AsIc+vuhzoIbs5tYfU9HyuJ/AX3suqPWczaw1cAowjOgTzc+4ENDKzz81snpmNqLXqwiOYc34S6Erg9pWLgNudc/7aKS8iQp5fQd3gIkJCdnNqDwn6fMzsbAKBfkZYKwq/YM75MeAu55wvMHnzvGDOOQHoDZwD1AO+NLNZzrkV4S4uTII55/OB+cBgIBeYYmbTnXN7w1xbpIQ8v+pyoNedm1PXnqDOx8xOBp4DLnDO7ail2sIlmHPOAyaWhXkmMMzMSp1z79RKhaEX7O/2dufcAeCAmU0DegJeDfRgzvla4I8usMCcb2ZrgC7A7NopsdaFPL/q8pLL9zenNrMkAjennlThmEnAiLJXi/sTzptT145qz9nMcoC3gKs9PFsrr9pzds61c861dc61Bd4Afu7hMIfgfrffBQaaWYKZ1Sdw8/VltVxnKAVzzusJ/IsEM2sOdAZW12qVtSvk+VVnZ+guBm9OHeQ53ws0AZ4um7GWOg93qgvynKNKMOfsnFtmZh8ACwE/8JxzrtLL37wgyJ/zA8AEM1tEYDniLuecZ9vqmtmrwFlAppkVAPcBiRC+/NJb/0VEokRdXnIREZEaUKCLiEQJBbqISJRQoIuIRAkFuohIlFCgi4hECQW6iEiU+H8pVz0Ck0hJUAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fit a model (code used from lab)\n",
    "lr_model = lr_search.best_estimator_\n",
    "lr_model.fit(X,Y)\n",
    "# predict probabilities\n",
    "probs = lr_model.predict_proba(X_valid)\n",
    "# keep probabilities for the positive outcome only\n",
    "probs = probs[:, 1]\n",
    "# calculate AUC\n",
    "auc = roc_auc_score(y_valid, probs)\n",
    "print('AUC: %.3f' % auc)\n",
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_valid, probs)\n",
    "# plot no skill\n",
    "pyplot.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6405e33",
   "metadata": {},
   "source": [
    "Seems pretty good on the validation set. But, how it will perform on the test set?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "3cecaef1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAsZElEQVR4nO3dd3gVxRrA4d+cJKTQQyehSq9CAGkqvQuEGkRARSJNQFDpTUTpTZqhSFWqKFIiVRClgxAgKAiCAUwQQ09CTjL3j3MMCSQhXFI2y/feZ5+cMzu7M8ONH8PszKzSWiOEEMJYLGldASGEEI+T4CyEEAYkwVkIIQxIgrMQQhiQBGchhDAgx5QuQDX0lOkg4jH3/X9L6yoIA3J1yKie9R5PE3P09qBnLi+lSM9ZCCEMKMV7zkIIkaqUYTvDT0WCsxDCXBwkOAshhPGYIzZLcBZCmIwMawghhAGZZJqDBGchhLlIz1kIIQzIHLFZgrMQwmRktoYQQhiQDGsIIYQBmSM2S3AWQpiMxRzRWYKzEMJczBGbJTgLIUzGwRwTnSU4CyHMRXrOQghhQDJbQwghDMgcsVmCsxDCZGS2hhBCGJA5YrMEZyGEycjybSGEMCB5ICiEEAZkjtgswVkIYTLScxZCCAMyxwJBCc5CCJORqXRCCGFAEpyFEMKAZMxZCCEMyByxWYKzEMJclPSchRDCeCQ4CyGEATmY5IGgSWYECiGEjVIqyUcS7vW+Uuq0UuqUUuprpZSLUspdKbVdKXXO/jN7rPxDlVLnlVK/KaUax0r3UkoF2M/NUkkoXIKzEMJUkis4K6U8gH5AFa11OcAB8AGGADu11sWBnfbvKKXK2M+XBZoAc5VSDvbbzQN8geL2o8mT2iHBWQhhKsnZc8Y29OuqlHIE3ICrQCtgqf38UqC1/XMrYJXWOkJrfRE4D1RTSuUDsmit92utNbAs1jUJkuAshDAVpZ7mUL5KqSOxDt//7qO1vgJMAS4D14BbWuttQB6t9TV7nmtAbvslHsBfsaoSZE/zsH9+ND1R8kBQCGEqTzNbQ2vtB/glcJ/s2HrDRYCbwFql1BuJFR1fEYmkJ0qCsxDCVCwq2QYEGgAXtdbXAZRS3wA1gWClVD6t9TX7kEWIPX8QUCDW9Z7YhkGC7J8fTU+UDGsIIUwlGcecLwPVlVJu9tkV9YFAYCPQzZ6nG/Cd/fNGwEcp5ayUKoLtwd8h+9DHHaVUdft9usa6JkHSc46He+Zs7Jy8GoC82XMRFR3N9Vs3AKjWtwWR1shnLmP3lLVkcnWjap/mAHiVqMAU35HU/aD9M99bpIzK5apQrHixmO/TZ0/DwyN/vHlreNVi/9Gfn6m8kcNGc/TwUTJlyoTFYmHoyMFUfLHiM93zeZBca1C01geVUuuAY4AVOI5tCCQTsEYp1R1bAG9vz39aKbUGOGPP30drHWW/XS9gCeAKbLUfiZLgHI9/79ykUk/bFMXRXQZyN+weU9d9EXPeweJAVHRUQpcnWe5sOWlStS7+h3c/871EynN2dmbNhlWpWub7HwygYeMG/PLzfj4ZM561365J1fLTI0syrhDUWo8GRj+SHIGtFx1f/vHA+HjSjwDlnqZsCc5J9OWH0/j39k0qFSvHsfMB3Ll/L07QDvDbQYuRb3IpOIjO9dvQr/XbZHBy4mDgcXp/Pozo6OjH7jl5zXxGdO73WHC2WCxM6D6MOhWr4+zkzJyNS/DbvBKlFLP7fsKrFapz8e+/sFgsLPZfzfqfNqfKn4GI6/69+wzo+z63b9/BarXSp19v6tavEyfP9evXGTxwCHfv3iMqKorho4ZSuUplfvl5P/Nnz+fBg0g8C3jy8fgxuGV0S7AsryqV+euy7YH/8iUr+PYb27+Kvdu15o2unQm7H8aHAwcTEhxMVFQ0vr3eoXHTxgnez8xk+fZzqIRnURoM9iE6OprRXQbGm6dUwWJ0fPU1ag1ojTXKypz3xtO5njfLd6x/LO/+wKN412pCnYo1uRN2Nya9e5NO3Lp3m2p9W5DBKQM/z9jAtqN78SpegcJ5C1DetwG5s+UkcNFuFvuvTrH2irgiIiLo4O0DgIenB5OnT2Ta51PJlCkToaGhdPXpRp16r8YJDls3+VOjVg169HyHqKgowsPDCQ0NZeH8hXyxaD6ubq58uXAJy5eu4N3evgkVzZ7deylWvBhnTp/huw0bWbFqGVpr3vDpSpUqXgQFXSFX7lzMnj8LgDt37qTsH4aBWUyyfFuC81NYu3dTvD3g2OpXqo1XifIcnmPrzbpmcCHk5o0E83/y1UxGdO7H4IWfxqQ18nqFCkVL0+4V23h0VrfMFPcoQu1yVVm7dxNaa4JDr7P7xP5kaJVIqkeHNSIjI/l8xmyOHTmGUhZCQq5z458b5MyVMyZP2fJlGDN8LFarlbr161KqdEmOHj7KhT8u0q3zWwBYIyOp8GKFeMucPmUGC+YvJLt7dsZ8MoqDBw5Rr0FdXN1cAajfsB7Hjh6n1ss1mTZ5OjOmzuSVV1+mcpXKKfgnYWzSc34O3QsPi/lsjbJisTyc7OKSwQUAhWLptnUMWzwhSffc/esvjHvzQ6qXfvgfk1KK9+aMZNuRPXHyNn8p3mEukUa2bNpK6L+hfLV2JU5OTjRt0JyIBw/i5PGq4sWi5Yv4ac9PjBgykm5vdyVLlsxUr/kSE6Z89sQy/htz/s+B/QfjzVeocCG+XruSfXv3MWvGbGrUrJ5oT9zMzBKcZSrd/+nP4CAqF7ON71cqVo4ieW3TG3ce30e7V5qTK1sOALJnzkbB3IkvBhr/1ed81KFXzPcfjuyhV4suODrY/u4s7lEENxdX9p06RNvazVBKkTtbTupUqJESTRNJdPfuXdzd3XFycuLwwcNcu3rtsTxXr1zF3T07bdu3oXXbVgSeCaR8xQr8euwEly9dBiAsLIxLf15KUpleVSqze+duwsLCCLsfxq4du6nsVYmQkOu4uLrQvGVzur7ZhcAzZ5O1relJMi/fTjPSc/4/rf9pC10btuX4/B84/NsJfr9yAYDAy+cY8eUktk34CouyEGmNpM/sEVwOuZLgvbYe2sX1WEMfC7d+ReG8nhyb548Crt/6l9aju7P+py3Ur1SbUwt28nvQBQ6ePc6te7dTuqkiAc1aNKV/7wG83r4zJUuVpEjRwo/lOXL4KEsXL8PR0RE3N1c+mTAOd/fsfPzpGIZ8OIxIe0+7T78+FCpc6Illli5TmpatW/JGx66A7YFgqTKl+GXfL0yfMgOlLDg6OTJ81LBkbWt6YvSgm1TKtg9HChbQ0DNlC3jOZHRx4174fdwzZ+PQ7E3UGuBNcOj1tK7WU7vv/1taV0EYkKtDxmeOrPnGvZLkmHNt5F7DRnLpOaczmz5ZSrZMWcjg6MS4lTPTZWAWIiXFfhaUnklwTmdkBaEQiUvORShpSYKzEMJUTBKbZbbGs+rn3Z0Avx2cWrCT/t7d45wb1O5d9PYgcmSJeYsN5YuU5peZ33FqwU5O+u3A2ck5zjXffbyYAL8dqVJ3kfL+vvY377zpi3eLNrR5rR0rl38FwEcDB9PB24cO3j40bdA8ZnHLf65dvUYNr1osXbwsLaqdrslsDUHZwiXp0bQT1d5rwYPISPw/W8HmQ7s4f+Uinrny0dDrZS4FP9xj28HiwIohs+gysR8nLwTinjkbkVEPN1Hyrt2Uu2H306IpIoU4ODow6KP3KV2mNPfu3aNTu85Ur1GdSdMmxuSZOnEamTJninPdlIlTqfVyrdSurimoeLdPTn+e2HNWSpVSSg22v5Rwpv1z6dSonNGVLliMA2ePExYRTlR0FHtOHsC7lu3VYNN7juGjBeOJPRumUZVXOXkhkJMXAgHbBkv/rTjM6OLGwLY9+GTlzNRviEgxuXLlonQZ238uGTNmpGjRIoSEhMSc11qz7YftNGn28JVyu3bsxsPTgxeKFU31+pqBWXrOiQZnpdRgYBW2nfwPAYftn79WSg1J+eoZ26k/f+OV8i/hnjkbrs4uNKtWjwK58vNajYZcufF3TBD+TwmPImg0/p+t4OjcrXwYa+HJuDc/ZOo6P+5HhD1ajDCJK1eucjbwN8pXeLg52bGjx8iRw51ChQsCEHY/jCWLltCz97tpVc10z2JRST6M7EnDGt2BslrrOBsYK6WmAaeBeNco29/DZVs7WiobeGZ85ooa0dnL55m4ei7bJ37N3bB7nLhwBmuUleGd+tFoyOuP5Xd0cKR22apU7duc+xFh7Jy0mqPnTnLjdijF8hdm4PyxFMrjGU9JIr27f+8+H/T/gA+HDiJTpodDGP6bf4jTa543ez6du3ZOdIc6kTij94iT6knBORrIDzy6tjSf/Vy8Yr+Xy+yLUBb7r2Kxv20znPFvDyY49B861/PmxBfbAPDMlY9j8/yp1rcFQf9cY0/AAW7cDgVgy6FdVC5Wnrth9/AqUZ6Ly/fj6OBI7mw52D1lrUybM4nIyEgGDfiAZi2aUb/hw/1RrFYrO3fs4uu1K2PSAk4GsH3bDmZMncmdO3ewKAvOzhnw6ewT361FPJ6X4DwA2KmUOsfDt8oWBIoBfVOwXulGrmw5uH7zBgVy5adNrabU6N+KWRsWxZy/uHw/Vfo048btUH44soePOvTC1dmFB5GRvFqhOtPXL2DLoV3M37QcgEJ5PNk0bokEZpPQWjN25McUKVqELm/GfTfowf0HKVKkMHny5olJ+3LF4pjP82bPx83NTQLzU3ougrPW2l8pVQKohu1V3grbywoPx3r9ynNt/Sg/cmTJTqTVSp/Zw7l591aCeW/evcW09Qs4PHszWmu2HNrNlkO7UrG2IrX9euxXNm3cTPESxWKmy703oC8vv1ob/63b4gxpiORhktgse2uItCF7a4j4JMfeGqVnNktyzAnsv8WwoVzmOQshTOW5GNYQQoj0xiSxWZZvPw2LxcKxef58P24JABVfKMP+WRttezrP2UzVki8+dk0Jz6Icn/9DzHHr28A4y7z7tnqLs4v3cGrBTia+MxyAmmWrcOKL7RyavYkX8hcGIGvGLPh/tiKlmyj+T6OHj6Fu7fq0bZn4g9xTAaepXK4K2394uEQ/oWtnTJ1J+9YdGDFkZEzapo2bYpaAi/g9F4tQRFz9vbsTePl8zPdJPYYzdvl0KvVszKilU5nUY/hj1/wedIFKPRtTqWdjvHo35X5EGBt+9gegTsWatKrZiArvNqRcj/pMWTcfsO3J0fZjX4Ytnkiv17oAMPKN/nz69exUaKX4f7T0fo25fon//xMVFcXMaTOpUSvuG2ziu/bOnTucOH6Ctd+uISoqmnO/nyM8PJyNG76ng4/M5EmMBOfnjEfOfDR/qT4Ltz7stWityeJmW1CQNWNmrt4ITvQe9SvV5o9rl2LeitLrtS5MWDWHB5G2t2H89zaUSGskrhlccHN2JdJqpWi+QnjkyMvekwdSomkiGXhV8SJL1qyJ5vl65SrqN6yPew73J15rsViIjLSitSYiIhxHR0eWLl5Gpzc64eTklOz1NxMJzs+ZGb1se2VERz98EDxg3hgm+47g8spDTPEdydBFib+w06dOS77e/V3M9xKeRXm5/EscmPU9P05dR5USFQH4bNVs/N6fyIA27zD7uyWMf+sjRi6dkjINE6kiODiE3Tt2075juyTlz5gxI/Ub1aNjm054eHiQKXMmTp86Td36dVK2oibwvCzfFtjeeh1y8x+OnQvg1VgvVe3VoivvzxvLN/u20P6VFiwaNIWGgzvFew8nRyda1mjE0EUPV7w7WhzInikr1fu9RtWSL7JmxDyKdq3JiT/OUKNfSwBeLv8SV28Eo4BVw+cSabUy6IuPCbn5T4q2WSSvyZ9Nof+gfjg4OCT5mre6v8lb3d8EYOzIj+ndtxffrNvA/p8PUKJkcXr0fCeFapvOGbxHnFQSnJOgVtmqtKzRiGbV6uGSwZksbplZPngWr9VoQP+5owBYu3cTCwdOTvAeTavW5dj5gDhBNeifv/lm31YADv/2K9E6mpxZ3fnn1r8xeUZ07kfHT3ozu+8njF42lcJ5CtDP+21GfDkphVorUsKZ02cYPGgoADdDb7Jv7z4cHByo16DuE689a3+TdqHChZj02WQWL1/E4EFDuPTn5ZgNk8RDRh+uSCoZ1kiCYYsnUOD1qhTpUgOf8X3Y9evPdJnYj6s3gmN60vUq1eLclYsJ3qNT3VZxhjQAvv3Fn3qVbHv2FvcoQgbHDHECc7dG7dl8cBc3797CzdmV6OhoonU0bs6uKdBKkZK2bN/E1h2b2bpjMw0aN2DYyKFJCswAcz6fS6/3ehFptRJl32JWWSyEh4enZJXTLaWSfhiZ9JyfQY9pHzGz91gcHRwJfxCB74zBAOTLkYeFAyfTfLjt9fWuzi409HqFd2fE3WV1sf9qFg+aSoDfDh5YI+k2eUDMOVdnF7o1bB+zu9209X6sH72AB5EP6PRpn9RpoEiyIR8M5ciho9y8eZNGdZvQq29PrJFWANr7JD7OHN+13m1bA7a9ncuWK0vu3LkAqFixAu1adaB4ieKULFUiRduUXpml5yzLt0WakOXbIj7JsXy7yqK2SY45R7qvN2wkl56zEMJUjD4LI6kkOAshTMUswxoSnIUQpiLBWQghDEiCsxBCGJAEZyGEMCB5ICiEEAYkPWchhDAgswRnWb4thDCV5Fy+rZTKppRap5Q6q5QKVErVUEq5K6W2K6XO2X9mj5V/qFLqvFLqN6VU41jpXkqpAPu5WSoJf4NIcBZCmEoy7+c8E/DXWpcCKgKBwBBgp9a6OLDT/h2lVBnABygLNAHmKqX+24ZwHuALFLcfT3ztugRnIYS5JFPXWSmVBXgFWASgtX6gtb4JtAKW2rMtBVrbP7cCVmmtI7TWF4HzQDWlVD4gi9Z6v7btl7Es1jUJkuAshDAVB4tK8qGU8lVKHYl1+Ma6VVHgOvClUuq4UmqhUiojkEdrfQ3A/jO3Pb8H8Fes64PsaR72z4+mJ0oeCAohTOVpHghqrf0AvwROOwKVgfe01geVUjOxD2EkVHR8RSSSnijpOQshTMWiVJKPJwgCgrTWB+3f12EL1sH2oQrsP0Ni5S8Q63pP4Ko93TOe9MTb8aQMQgiRniTXA0Gt9d/AX0qpkvak+sAZYCPQzZ7WDfjvLRobAR+llLNSqgi2B3+H7EMfd5RS1e2zNLrGuiZBMqwhhDCVZO5xvgesVEplAC4Ab9mLWKOU6g5cBtoDaK1PK6XWYAvgVqCP1jrKfp9ewBLAFdhqPxIlwVkIYSoOluQLz1rrX4Eq8Zyqn0D+8cD4eNKPAOWepmwJzkIIU0nCWHK6IMFZCGEqZlm+LcFZCGEqZpnlIMFZCGEqMqwhhBAGJMMaQghhQA4SnIUQwnhkWEMIIQxIgrMQQhiQjDkLIYQBSc9ZCCEMyByhWYKzEMJkHJNxb420JMFZCGEqMuYshBAGJGPOQghhQOYIzRKchRAmIz1nIYQwoOTcbD8tSXAWQpiKOUKzBGchhMnIbA0hhDAgGXMWQggDkuCcRGH+v6d0ESIdcm1SIq2rIAxIbw965nvIsIYQQhiQgzLHI0EJzkIIU5FhDSGEMCBlkjWCEpyFEKYiY85CCGFAMqwhhBAGpEyyRlCCsxDCVGRvDSGEMCB5ICiEEAYkY85CCGFAMltDCCEMyCIPBIUQwngs8kBQCCGMxyIPBIUQwnhkzFkIIQzILLM1zDE4I4QQduop/pek+ynloJQ6rpTaZP/urpTarpQ6Z/+ZPVbeoUqp80qp35RSjWOleymlAuznZqkkdO8lOAshTMWiLEk+kqg/EBjr+xBgp9a6OLDT/h2lVBnABygLNAHmKqUc7NfMA3yB4vajyRPbkdTaCSFEepCcwVkp5Qk0BxbGSm4FLLV/Xgq0jpW+SmsdobW+CJwHqiml8gFZtNb7tdYaWBbrmgTJmLMQwlSSecx5BvARkDlWWh6t9TUArfU1pVRue7oHcCBWviB7WqT986PpiZKesxDCVJ5mzFkp5auUOhLr8I25j1ItgBCt9dEkF/04nUh6oqTnLIQwlafpOWut/QC/BE7XAloqpZoBLkAWpdQKIFgplc/ea84HhNjzBwEFYl3vCVy1p3vGk554O5LcCiGESAeUsiT5SIzWeqjW2lNrXRjbg75dWus3gI1AN3u2bsB39s8bAR+llLNSqgi2B3+H7EMgd5RS1e2zNLrGuiZB0nMWQphKKmwZOgFYo5TqDlwG2gNorU8rpdYAZwAr0EdrHWW/phewBHAFttqPRElwFkKYSkpstq+1/hH40f75BlA/gXzjgfHxpB8Byj1NmRKchRCmIntrCCGEAcneGkIIYUBPetCXXkhwFkKYigxrCCGEAT3FnhmGJsFZCGEqMuYshBAGJMMaQghhQPJAUAghDCgVVgimCgnOQghTkTFnIYQwIJmtIYQQBiQPBIUQwoBkWEMIIQxImWSbegnOj6hUzovixYvFfJ8+ezoeHvnjzVvdqyYHjv7yTOWNHDaK/b8cYMu2TWTIkIHQ0FBeb9+ZrTu2PNN9Rcpwz5yNnZNXA5A3ey6ioqO5fusGANX6tiDSGvnMZeyespZ87rkJfxDB3fB7vD1lEL8HXXjm+z4vpOdsUs7OzqzZsDpVy3SwOPDtN9/SwadDqpYrnt6/d25SqWdjAEZ3GcjdsHtMXfdFzHkHiwNR0VEJXZ5knSe8x9HfT9KjWWcm+46g1ai3n/mezwsHeSD4fLh/7z79+77P7du3sVqt9O3Xm7r168bJc/36dT4aOJh7d+9hjYpixKhhVK5SmV9+3s+82fN48CCSAgU8+Xj8WNwyuj1WRueur7N86UratGvz2Lkli5ay7YdtPHgQSb36den9Xi8Avpjnx5ZNW8mbNw/ZsmejTJkydHu7a8r8IYhEffnhNP69fZNKxcpx7HwAd+7fixO0A/x20GLkm1wKDqJz/Tb0a/02GZycOBh4nN6fDyM6OjrBe+8NOMCANt0BmNRjBE2r1kEDn6ycyZo935PXPTerh88lS8bMOFoc6DVrGPtOHUqNZhuWzHM2qYiICDp4dwQgv6cHU6ZPYvrnU8mUKROhoaF08elGnXp14vzTacumrdSsVZMePd8hKiqK8PBwQkNDWTB/AV8s+gI3N1cWL/ySZUuX07P3u4+VmS9fXipVrsSmjZt5te4rMem//Lyfy5cvs3L1CrTW9OszgKNHjuLi4sLObTtZvf5roqKi8GnbiTJlyqT8H45IUAnPojQY7EN0dDSjuwyMN0+pgsXo+Opr1BrQGmuUlTnvjadzPW+W71if4H1fq96QgItnaVO7GS++UIaKPRuRM4s7h+dsZm/AQV6v15ofju7h068+x2Kx4ObsmlJNTDdkWMOkHh3WiIyMZNaM2Rw7cgyLUoSEhHDjnxvkzJUzJk+58mUZPXwsVquVuvXrUqp0SY4cPsqFPy7yZuc3Y+5T4cUKCZb7ju/b9O8zgJdffTkmbf/P+9n/8346tvEB4P79MC5dusz9e/epU68OLi4uALwSK6CLtLF276ZEe8AA9SvVxqtEeQ7P2QyAawYXQm7eiDfvyiGfE/YgnD///ov35oxkYFtfvt79HdHR0YTc/Ic9Jw9QtWRFDv92gsWDpuLk4MS3v/hz4o8zyd629EYeCD4ntmzaSui/oXy9diVOTk40bdCMiAcP4uTxquLF4uUL+WnPPoYPGcGbb3clc5YsVK/5EhOnTEhSOQULFaRk6ZJs898Wk6a15u0eb9O+Y7s4eZcvXfHsDRPJ6l54WMxna5QVS6z32LlksP0lqlAs3baOYYuf/Dvx35jzfxLqDf4UcJBXBrWlebX6LB88i8lr5iXaE38emKXnbI6/YlLQ3bt3cXfPjpOTE4cOHubq1WuP5bl65Sru7u60bd8G77atCTxzlgoVy/PrsRNcvnQZgLCwMP7881KiZb3j+w7LvlwW871m7Zp8+8133L93H4Dg4BBu3PiXSpUrsefHvURERHD/3n1+2rMvGVssntWfwUFULmZ7l2elYuUokrcAADuP76PdK83JlS0HANkzZ6Ngbo8k3XNvwEE61nkNi8VCzqzuvFL+JQ6d/ZWCuT0ICf2HhVu/YtHWr6lcvHzKNCodsaCSfBiZ9JyfoFmLpvTr3Z9O7V+nZKmSFCla5LE8Rw4fYcniZTg6OuLm5sYnE8bh7u7Ox5+OZciHQ3nwwDa9qm+/3hQuXCjBsooVf4FSZUpz9kwgADVr1eDihYt0eb0bAG5urnw6cTzlypelTt1Xae/dkXz581G2bBkyZc6UAq0X/4/1P22ha8O2HJ//A4d/O8HvV2zT4AIvn2PEl5PYNuErLMpCpDWSPrNHcDnkyhPvuWHfVmqU9uLE/G1o4KMF4wkOvU7Xhu34sH1PIqOs3A27R9dJA1K2cemAWZZvK611ihYQHnU/ZQt4Tt2/dx+3jG6EhYXxdtfujBo7ktJlSqd1tZLMtUmJtK6CMCC9PeiZu7O7rm5Ncsypl7+pYbvP0nNOpz4eM44L5y8Q8eABLVu1SFeBWYiUJA8ERZqaMPmztK6CEIZkkQeC4lGjho+hTu16tGn5cHbF2cDfeMOnKx28O9Kp/esEnDwFQMDJU3Tw7kgH74609+7Azh270qraIgX08+5OgN8OTi3YSX/v7nHODWr3Lnp7EDmyZAfA0cGRJR9O56TfDs4s2s0Qnz4xeX3qtuKk3w5OfLGdrZ+uiLlGJEw9xf+MTIJzMmrl/Rrz/ObESZs+dQY9e/uyZsNqevftxYypMwDbw7+v1q5kzYbVzPWbw7gxn2C1WtOg1iK5lS1ckh5NO1HtvRZUfLcRLao3oJiH7UGyZ658NPR6mUvBQTH527/SAmenDFTwbYBX76a82/wNCuXxxMHiwMxeY6n7QXsqvtuQkxcC6dvqrbRqVrqhlEryYWQSnJORVxUvsmTNGidNKcXde/cA27S8XLlzAeDq6oqjo21UKSLigeF/UUTSlS5YjANnjxMWEU5UdBR7Th7Au1YTAKb3HMNHC8YT+0G8RpPRxQ0HiwOuGVx4YI3k9v27MQEko4ttyX+WjJm4eiM4TdqUnliUJcmHkcmYcwr7aMgH9OrRh2mTpxMdHc2ylUtizp08EcDoEWO4dvUa4yd+EhOsRfp26s/fGP/WYNwzZyPsQTjNqtXjyO8nea1GQ67c+JuTFwLj5F+3dzOtajTi2upjuDm78v78sYTeuQlAr1nDCPDbwb3w+5y7cpE+nw9PgxalLxaT9Dn/71YopRL895VSylcpdUQpdWTRgsX/bxGmsGbVWj4cMohtu/z5cPAHjBk5NuZchYrl2fD9er5as4JFCxYTERGRhjUVyeXs5fNMXD2X7RO/xv/TFZy4cAZrlJXhnfoxasmUx/JXK/UiUdHR5PfxokjXGgxq50uRvAVxdHCk12tdqNSrCfl9vDh58SxDffqmQYvSFxnWgLEJndBa+2mtq2itq3Tv8Xxvdfj9d5uo37A+AI2aNORUwOnH8hR9oSiurq6cP3c+tasnUshi/1V49W7Kq4Pa8e+dm/wZHESRvAU48cU2Li7fj2eufByb50+e7Ll4vV5r/I/8iDXKyvWbN/j59GGqlKjAiy+UBeDCNdvK0jV7vqdm2Spp2ax04bl4IKiUOpnAEQDkSaU6pmu5cufiyOGjABw6cIiChQoCEBR0JeYB4NUrV7l08U/yJ7Cpv0h//luiXSBXftrUasqy7evI0+FFinSpQZEuNQi6fo3KvZoQHHqdyyFXqfdiTQDcXFypXroyZ//6gys3/qZMweLkzOoOQMPKLxN4+VyatSm9MEvP+UmDnHmAxkDoI+kKeLZXgJjQ4A+GcOTQUW7evEnDuo3p1bcno8aOZNJnk4mKspIhgzOjxo4A4Pix4yxe8CVOjo4oi4VhI4eRPbtMkzKL9aP8yJElO5FWK31mD+fm3VsJ5p3z3RK+/HAapxbsRCnFlz+sIeCibVx67Irp7J22nkirlUvBQbw5+f3UakK6ZZYx50SXbyulFgFfaq0f21lHKfWV1vr1JxUgy7dFfGT5tohPcizfPvLPL0mOOVVy1jRs9znRnrPWunsi554YmIUQIrUZfSw5qWTulhDCVIw+lpxUEpyFEKZilp6zOUbOU9nPP/1My2atadG4JfHN47596zYD3htIu9YdeL3jG5x7ZIpcVFQUHdr40LdXv5i06VNn0q51B4YPGRGT9v3GTaxc/lXKNUQ8M4vFwrF5/nw/bgkAFV8ow/5ZG217Oc/ZTNWSL8Z73YA273BqwU4C/Hbw1bDZODs5J3p9zbJVOPHFdg7N3sQL+QsDkDVjFvw/k7fiPOq5mEonHhcVFcWnn0xg7hez2fD9evy3+PPH+T/i5Fnot4hSpUqy7ts1jP9sHJM+nRzn/MrlX1H0hYeb9t+5c4cTx0+w7ts1REdFc+73c4SHh7Nxw/d08GmfKu0S/5/+3t0JvPzwL99JPYYzdvl0KvVszKilU5nU4/EVfflz5KVf67ep0qc55X0b4GBxwKduy0SvH9TuXdp+7MuwxRPp9VoXAEa+0Z9Pv56dCq1MX5Jr+bZSqoBSardSKlApdVop1d+e7q6U2q6UOmf/mT3WNUOVUueVUr8ppRrHSvdSSgXYz81SSRh7keD8lE4FnKJAwQJ4FvDEKYMTTZo25sddP8bJc+GPC1SrXg2AIkWLcPXqVW78Y3uRZ/Dfwfy0Zx/ebb1j8lssFiIjI9FaEx4RgaOjI0sWL+X1N3xwcnJKtbaJp+ORMx/NX6rPwq0P/3WjtSaLm+2tNFkzZk5wLwxHB0dcnV1wsDjg5uwaky+h6yOtkbhmcMHN2ZVIq5Wi+QrhkSMve08eSMkmpkvJ2HO2AoO01qWB6kAfpVQZYAiwU2tdHNhp/479nA9QFmgCzFVKOdjvNQ/wBYrbjyZPKlzGnJ9SSHAIefM+XH+TO2+emG1A/1OiZAl27thJZa9KBJw8xbWr1wgODiZHzhxMmjCZ9z/ozz37ewEBMmbMSING9enYxodq1auRKXMmTp86Q8/e76Zau8TTm9HLtolRZteHrwgbMG8MP3y2kim+I7FYLNTs3+qx667e+Jsp677g8sqDhEWEs+3oXrYf3Zvo9Z+tmo3f+xMJiwiny8T+TPEdwciljy8FF8n3QFBrfQ24Zv98RykVCHgArYA69mxLgR+Bwfb0VVrrCOCiUuo8UE0p9SeQRWu9316/ZUBrYGti5UvP+SnFNy380V+Ft3u8xe1bd+jg3ZGvV66iVOmSODg4sOfHvbi7u1OmbJnH7vFW9zdZs2E1HwwexJxZc+ndtxffrPuGD9//CL/5C1KmMeL/1vyl+oTc/Idj5wLipPdq0ZX3542lYOdqvD9vDIsGPR5As2XKSqsajSjSpQb5fbzI6OJK5/ptEr3+xB9nqNGvJfU+7EDRfAW5eiMYBawaPpflg2eRO1vOFG9zevE0PefY+wDZD99476lUYaAScBDIYw/c/wXw3PZsHsBfsS4Lsqd52D8/mp4oCc5PKU/e3Pz998N/qob8HUxu+zag/8mUKRPjPh3Lmg2rGT9hHKH/huLh6cGvx37lx917aNqgGYMHDeHwwcMM/SjumGTgmbMAFCpciO+/28Tk6ZM4f+48l57w5m6RumqVrUrLGo24uHw/q4bPod6LtVg+eBbdGrXjm31bAFi7dxPV4nkg2KBybS7+/Rf/3PoXa5SVb/ZtpWYZL4AkXT+icz/GrZzJ6C4DGb1sKit2fkM/7+d7D5vYnmb5dux9gOyHXzz3ywSsBwZorW8nVnQ8aTqR9ERJcH5KZcuV5fKlywQFXSHyQST+W3/g1bp14uS5ffsOkfY3bn+zbgOVq1QmU6ZM9B/Yj+27f2Drji1MnDqBqi9V5bNJ4+NcO+fzufR+rxdWq5Xo6GjANiYdHh6eGs0TSTRs8QQKvF6VIl1q4DO+D7t+/ZkuE/tx9UYwr1aoAUC9SrU4d+XiY9deDrlK9dKVcHV2AaB+pdoxDxWfdH23Ru3ZfHAXN+/ews3ZlejoaKJ1NG7OrinZ3HQlOWdrKKWcsAXmlVrrb+zJwUqpfPbz+YAQe3oQUCDW5Z7AVXu6ZzzpiZIx56fk6OjI0OGD6dWjN9HR0bT2bkWx4i+wZtVaADr4tOfihQuMGDISi4MDRV8oythxo5N07107dlOuXFly57b9K6lCxQq0bdWeEiWKU7JUyRRrk0g+PaZ9xMzeY3F0cCT8QQS+MwYDkC9HHhYOnEzz4V05dPY4637awrG5/lijrBz/4zR+W1Ymej2Aq7ML3Rq2p9EQ2+Lcaev9WD96AQ8iH9Dp0z6PV+Y5lVyb6NtnVCwCArXW02Kd2gh0AybYf34XK/0rpdQ0ID+2B3+HtNZRSqk7Sqnq2IZFugKfP7H8xPbWSA6yt4aIj+ytIeKTHHtr/H7rVJJjToms5RIsTylVG/gJCACi7cnDsAXYNUBB4DLQXmv9r/2a4cDb2GZ6DNBab7WnVwGWAK7YHgS+p58QfCU4izQhwVnEJzmC87lbp5Mcc4pnLWvYlSgyrCGEMBXZW0MIIQxJgrMQQhiO0d+qnVQSnIUQpmL0DY2SSoKzEMJUZMxZCCEMSHrOQghhQBKchRDCgGRYQwghDEhmawghhAHJsIYQQhiSBGchhDAcc4RmCc5CCJORB4JCCGFIEpyFEMJw5IGgEEIYkFmGNcwxIVAIIUxGes5CCFORYQ0hhDAgCc5CCGFAMuYshBAixUjPWQhhKjKsIYQQhiTBWQghDMccoVmCsxDCZMzyQFCCsxDCVGTMWQghDEmCsxBCGI5ZhjVknrMQQhiQ9JyFEKYiY85CCGFIEpyFEMJwLCYZc5bgLIQwGQnOQghhOOYIzRKchRCmY47wLMFZCGEqZpnnLMFZCGEqZplKp7TWaV2H54ZSyldr7ZfW9RDGIr8XIj6yQjB1+aZ1BYQhye+FeIwEZyGEMCAJzkIIYUASnFOXjCuK+MjvhXiMPBAUQggDkp6zEEIYkARnIYQwIAnOqUQp1UQp9ZtS6rxSakha10ekPaXUYqVUiFLqVFrXRRiPBOdUoJRyAOYATYEyQCelVJm0rZUwgCVAk7SuhDAmCc6poxpwXmt9QWv9AFgFtErjOok0prXeC/yb1vUQxiTBOXV4AH/F+h5kTxNCiHhJcE4d8e3EInMYhRAJkuCcOoKAArG+ewJX06guQoh0QIJz6jgMFFdKFVFKZQB8gI1pXCchhIFJcE4FWmsr0Bf4AQgE1mitT6dtrURaU0p9DewHSiqlgpRS3dO6TsI4ZPm2EEIYkPSchRDCgCQ4CyGEAUlwFkIIA5LgLIQQBiTBWQghDEiCsxBCGJAEZyGEMKD/AaWBHUTBqjFWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# review a confusion matrix (code used from lab)\n",
    "lr_predicted = lr_model.predict(X_valid)\n",
    "lr_matrix = confusion_matrix(y_valid, lr_predicted) \n",
    "# print(lr_matrix)\n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in lr_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in lr_matrix.flatten()/np.sum(lr_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(lr_matrix, annot=labels, fmt='', cmap='Greens')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bd4e17a",
   "metadata": {},
   "source": [
    "Type I error (false alarm) = FP / FP + TN = 248 / 248 + 9490 = 0.02547 (2.5%)\n",
    "\n",
    "Type II error (miss rate) = FN / FN + TP = 173 / 173 +9498 = 0.01789 (1.8%)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a53cfc5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.98      0.97      0.98      9738\n",
      "         1.0       0.97      0.98      0.98      9671\n",
      "\n",
      "    accuracy                           0.98     19409\n",
      "   macro avg       0.98      0.98      0.98     19409\n",
      "weighted avg       0.98      0.98      0.98     19409\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "lr_report = classification_report(y_valid, lr_predicted) \n",
    "print(lr_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d8e948c",
   "metadata": {},
   "source": [
    "### Naive Bayes Further Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "0a3ce6fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('accuracy', 0.98117, 0.00213)\n",
      "('neg_log_loss', -0.39884, 0.03902)\n",
      "('roc_auc', 0.99375, 0.00094)\n"
     ]
    }
   ],
   "source": [
    "print(scoring_cv('accuracy',nb_search.best_estimator_))\n",
    "print(scoring_cv('neg_log_loss',nb_search.best_estimator_))\n",
    "print(scoring_cv('roc_auc',nb_search.best_estimator_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "433191ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.993\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkd0lEQVR4nO3deXiU5bnH8e+dhAQIIQHCTsISQFZRCKAooiiKVKW29rgdrUtFW7V1aSuttbW155yeY8WlainuSxWtoFLFBTfAhQpYCIsCYY8g+w4hycxz/piAIQxkgMlMnpnf57q4yMy8mdyvib/cPPPM/ZpzDhER8V9KvAsQEZHoUKCLiCQIBbqISIJQoIuIJAgFuohIgkiL1xfOzc11HTp0iNeXFxHx0uzZszc655qHeyxugd6hQwdmzZoVry8vIuIlM1t5qMe05CIikiAU6CIiCUKBLiKSIBToIiIJQoEuIpIgagx0M3vSzNab2fxDPG5m9pCZFZtZkZn1jX6ZIiJSk0g69KeB4Yd5/FygS+WfUcBfj70sSSpTfgcPnRj6u6rVn8P0+0J/iySKWvy5rnEfunNumpl1OMwhI4FnXWgO7wwzyzGz1s65tdEqUg7hv9tB2Y4qd6QADtr0hVEfHHz8rKfhy9ehYS7s3gjdR0LhVaH7ZzwKZlC2G3asgVZ9Qs+x73O6jwz9EG5bBZYKuV2g1fGwdm7o8wb+OPRcABOugyVTwAHlO6BFLzj+P2DxW9DlHOh9ETw2NPR10hpCxe7Q533yAKz8FDqdDtu/hqKXIBiAlFQ4/mJo3La2/kuKxERgWwkpRS9jLghpGfDDSZA3IGrPb5HMQ68M9Decc73CPPYG8Cfn3MeVt98H7nDOHfSuITMbRaiLJz8/v9/KlYfcH390ft8UXKDyi6VA6xPDB5vvJlwH814+/DEZOdDpNMCBc7BlBawLs2qW2QJ2rQ//HJYGriLyuiz12//+x8QI/TYId7+InxzgcBiVP8mWCkPvhMG3H9HzmNls51xhuMei8U7RcP+Xhf0t4ZwbB4wDKCwsjO6VNe7OrvbFgrBmNvxPPnQcXL2QA8usfjtaxxzwyzLcfTUdE+ZzNi2G3Zup0d6tsHFx6BebpYQCPZzdmw79HEcS5gDZbWHneqgoPbLPq6r9ILj6rdA/R5+5AAJlkJoe9U5GJFZKywM8+P4Sxk1bxmkNlvM4fyA1WB76ue4wuOYnOALRCPQSIK/K7XbAmig8b+TuaXHox/Zug83LQx9b1d89dsBf394Od4wdwzFVjq3xGAvdd8DTVzumdDsRadPvwH+dzHoa3vjZwcf1+v6hu/3s/NASS6ROvR1WflLzvx7qN4HSLQffb6lw1u9DH+cNCIX4iumhH3qFuXhq1HOzmbZ4Az/o147ffGcYqZv619rPdTQCfRJwk5mNBwYC22K+fh7Ye+jHqgeb7w673HKYNfR969vh1tDbnxK9NfR9X6e4cg29bEfoufpe+e3zFF4F9/eGbashOw8ueiL8D3jeAAW5eGnn3grSUoz69VL58ZACrhvckcFdKudpNay9n+sa19DN7EXgdCAXWAf8DqgH4Jwba2YGPExoJ8xu4Opw6+fVFRYWuqgN56q+3LJPooX5PuFC/e5t8alFRA4wdfEGfj1xHt89sQ2/OKdb1J//mNbQnXOX1vC4A248ytpqTyIH3PcfC/0RkTpj6+4y7nnjSyZ8UUJB80yGdjvMUnAtidv4XBGRRPFJ8UZ+Nn4OW3eXcdMZnblpaGfq10uNeR3+B7redCIicdasUTp5TRvwzDX96dnmEEvAMeB/oL9xW5g7Y/+bUUSSh3OOV2aXsGDNdu6+oCfdWjVm4o8HYQfsgIs9/wN9w5cH33fKzbGvQ0SSwurNu/n1q/OYvmQjAzo0pbQ8QP16qXEPc0iEQA+G2aUz7Pexr0NEElog6Hj2sxX839uLSDG457u9uHxAPikp8Q/yffwP9Mxc2LWuyu2W8atFRBLW5l1ljJmymIGdmvJfF/ambU6DeJd0EP/noZ/x68PfFhE5SuWBIP+YtZpg0NE8K4M3bx7MU1f1r5NhDokQ6Cs/OfxtEZGjMK9kG+f/5WN+8UoR04s3ApDfrGGdWCs/FL+XXB4eABsXHXhf8ZT41CIiCaG0PMAD7y3hsenLaJaZzt+u6MeQrs3jXVZE/A30v/QPTR6srmx37GsRkYRx3bOzmL5kI5f0z+NXI7qT3aBevEuKmL+BvmlJ+PsPN6hLRCSMHaXl1EtNoX69VG48ozM3DCnglM658S7riPm7ht64Xfj7tctFRI7Ah1+t55z7p/HQ+6Em8aROzbwMc/A50Ec+VPmBsf80MlvCL8Isw4iIVLN5Vxm3vjSHq5+eSWZGGmf18L8Z9HfJJRgM/X3tu5qZLSJHZPqSDdwyfg7b9pTz0zO7cOMZBWSk+T8yxN9A33/tUP+/CSISWy2y6tMxN5M/XtiLbq0ax7ucqPF3ySVYGegp/p6CiMSGc47xn6/irtdCF0o/rlUW/7jh5IQKc1CHLiIJbtWm3YyeWMSnSzdxUqe6NUwr2vwN9P0dugJdRA4WCDqe+mQ5f353EWkpKfz3hb25pH9enRqmFW3+BrqrfFHUtOQiIgfbvKuMB99fwikFufzxwl60zq6b81eiKQECXR26iISUVQR57d9fc1G/djTPymDyTwfTrkmDhFxeCcffQNeSi4hUMXf1Vn75ShGL1u2gVXZ9TuvanLymDeNdVkz5G+j7XxTVkotIMttTFmDMlEU88fFyWmTV5/ErCznNk2Fa0eZvoKtDFxFCw7Q+Lt7IpQPy+dWIbjSu788wrWjzN9C1bVEkaW0vLSe9cpjWzUM785MzChhU4Of8lWjyd71CHbpIUnr/y3WcPWYaD1YO0xrYqZnCvJLHHbp2uYgkk0079/L7fy5k0tw1dGuVxfCereJdUp3jb6CrQxdJGtMWb+CWl+awo7ScW8/qyo9PLyA9zd8Fhtrib6Brl4tI0miVXZ/OzRvxxwt70bVlVrzLqbP8TUN16CIJKxh0vPCvVdz56jwAurbM4uUbTlaY1yABOnQFukgiWbFxF6MnFjFj2WZO7tRs/zAtqZm/ga4OXSShBIKOJz9ezn1TFlEvJYU/fa83F/fPS5q37UdDREsuZjbczBaZWbGZjQ7zeLaZ/dPM5prZAjO7OvqlVqNdLiIJZfOuMv7ywRJO7dycKbcN4ZIB+QrzI1Rjh25mqcAjwDCgBJhpZpOccwurHHYjsNA5d76ZNQcWmdnfnXNltVI1qEMXSQB7KwJM/OJrLi7MCw3T+tlg2uYkzzCtaItkyWUAUOycWwZgZuOBkUDVQHdAloW+C42AzUBFlGs9kAsABvrGi3jp36u2cMeEIhav20nbnAac1rU57Zok1zCtaIsk0NsCq6vcLgEGVjvmYWASsAbIAi52bt+ayLfMbBQwCiA/P/9o6v1WMKDuXMRDu8squO/dxTz5yXJaNa7PU1f1T9phWtEWSaCHa4FdtdvnAHOAoUABMMXMpjvnth/wSc6NA8YBFBYWVn+OI+MCWj8X8dCoZ2fzcfFG/vOkfO4Y3o2sJB6mFW2RBHoJkFfldjtCnXhVVwN/cs45oNjMlgPdgM+jUmU46tBFvLFtTzkZaaFhWj89sws3D+3MwE7N4l1Wwolkl8tMoIuZdTSzdOASQssrVa0CzgQws5bAccCyaBZ6EOf0LlERD0xZuI6z75/KA++FhmkN6NhUYV5LauzQnXMVZnYT8A6QCjzpnFtgZjdUPj4WuAd42szmEVqiucM5t7EW69aSi0gdt3HnXu6etIA3itbSrVUWI3prmFZti+iNRc65ycDkaveNrfLxGuDs6JZWg2AAUtShi9RFHy1azy0vzWH33gC3D+vKDacXUC9V/7/WNn/fKaoOXaTOapPTgONaZvHH7/aii+avxIy/vzL1oqhInREMOp6bsZJfTfx2mNZL15+sMI8xdegickyWbdjJ6Anz+HzFZgZ3ydUwrTjyN9CDQXXoInFUEQjy2PTl3P/eYuqnpXDvRcdzUb92ett+HPkb6C6gbYsicbRldzljpy7ljOOac8/IXrRoXD/eJSU9fwNda+giMbe3IsArs0u4tH8+zbMyeOtng2mT0yDeZUklfwNda+giMTV7ZWiYVvH6nbRvmsmpXXIV5nWMv4GuDl0kJnbtreDP7y7i6U9X0Ca7Ac9cM4BTu+TGuywJw99Ad0F16CIxMOq5WXxSvIkfntyeXwzvRqMMf2Mj0fn7ndE7RUVqzbbd5WTUCw3TuuWsrtxyFvTv0DTeZUkN/E1EraGL1Iq356/lrPuncv97i4FQkCvM/eB5h65AF4mW9TtK+d3rC3hr/jf0aN2Y849vE++S5Aj5G+jq0EWi5sNF67ll/Bz2lAf4xTnHMeq0Thqm5SF/A10dukjUtMtpQM82jfnDyF50btEo3uXIUfL3V7B2uYgctWDQ8cynKxg9oQiALi2zeOG6kxTmnvO7Q09Lj3cVIt5ZumEnd7xSxKyVWzita3MN00og/ga6OnSRI1IeCDJu2jIefH8JDeql8ucf9OH7fdtqmFYC8TjQNZxL5Ehs21POuGnLOKt7C+6+oCctsjRMK9H4G+h6UVSkRqXlAf4xazWXD2xPbqMM3r5lMK2zNX8lUfkb6Nq2KHJYM1ds5o5Xili2cRcdcxtxapdchXmC8zfQdYELkbB27q3g/97+imc/W0m7Jg147loN00oW/ga61tBFwhr17Cw+W7aJq0/pwM/PPo5MDdNKGv5+p7WGLrLf1t1lZKSl0iA9ldvP7goY/do3iXdZEmP+trhaQxcBYPK8tZw1ZioPVA7T6te+qcI8SalDF/HU+u2l3PX6fN5ZsI7ebbMZeULbeJckceZvoOuNRZLEPvhqHbeMn8PeiiCjz+3Gj07tSJqGaSU9fwNdF7iQJJbftCF98nL4/QU96dRc81ckxN9E1Bq6JJFA0PHkx8v55StzAejcIovnrh2oMJcDeN6hK9Al8S1Zt4M7JhTxxaqtnHGchmnJofkb6OrQJcGVVQT529Sl/OWDYjIzUnng4hMYeUIbDdOSQ4poycXMhpvZIjMrNrPRhzjmdDObY2YLzGxqdMsMQ+8UlQS3vbScJz5Zztk9WzLltiF890RNRpTDq7FDN7NU4BFgGFACzDSzSc65hVWOyQEeBYY751aZWYtaqvdb6tAlAZWWB3hp5mquOCk0TOudW06jZWNNRZTIRLLkMgAods4tAzCz8cBIYGGVYy4DJjrnVgE459ZHu9CDaJeLJJh/LdvE6InzWL5xF51bNOKUzrkKczkikSRiW2B1ldsllfdV1RVoYmYfmdlsM7sy3BOZ2Sgzm2VmszZs2HB0Fe+jDl0SxI7Scn7z2jwuHjeDimCQv/9oIKd01jAtOXKRdOjhFu1cmOfpB5wJNAA+M7MZzrnFB3ySc+OAcQCFhYXVn+PIaJeLJIhRz85mxvJNXHtqR24/uysN0/3dqyDxFclPTgmQV+V2O2BNmGM2Oud2AbvMbBrQB1hMbVGHLh7bvKuMBvVCw7R+fs5xmEHffM1fkWMTyZLLTKCLmXU0s3TgEmBStWNeBwabWZqZNQQGAl9Gt9QqXGVzr/G54hnnHJPmruGsMVO5f/8wrSYKc4mKGjt051yFmd0EvAOkAk865xaY2Q2Vj491zn1pZm8DRUAQeNw5N7/Wqg4GQn9ryUU88s22Un7z2nze+3Idfdpl872+GqYl0RXRYp1zbjIwudp9Y6vdvhe4N3qlHa6gykBXhy6eeP/L0DCt8mCQO0d055pTO5Kaoj3lEl1+vvqiDl08075ZJn3bN+H3F/SkQ25mvMuRBOVni7u/Q1egS90UCDoen76M21/eN0yrEc9cM0BhLrVKHbpIlC1et4NfvlLEnNVbGdqthYZpScz4GeguGPpbHbrUIWUVQf760VIe/nAJWfXr8eAlJ3BBHw3TktjxM9DVoUsdtL20nKc/Xc6I3q357Xk9aNYoI94lSZLxM9C1y0XqiD1lAV78fBU/HNRh/zCtFpq/InHiZ6CrQ5c64NOlGxk9YR6rNu/muFZZnNI5V2EuceVnoGuXi8TR9tJy/mfyV7z4+SraN2vIi9edxMkFzeJdloinga4OXeJo1LOz+Hz5Zq4/rRO3nNWVBun6OZS6wc9A1y4XibFNO/fSMD2NBump/HJ4N1LN6JOXE++yRA7g56uK6tAlRpxzvD7n6wOGafXNb6IwlzrJ0w5du1yk9q3dtoffvDqf979azwl5OVzUr128SxI5LD8DXR261LIpC9dx60tzCAQdd53Xg6sGddAwLanz/Ax07XKRWtYxN5PCDk34wwW9yG/WMN7liETEzzULdegSZRWBIOOmLeW2l+YAoWFaT189QGEuXvG0Q9cuF4meL9du544JRRSVbGNYj5YapiXe8jzQ/fwHhtQNeysCPPLhUh79sJichvV45LK+jOjdSsO0xFt+Bvr+JRcFuhy9naUVPD9jJRf0acNd5/WgSWZ6vEsSOSZ+BrpeFJWjtLusghf+tYqrT+lIs8phWs2zNBVREoOfga4XReUofFK8kdETi1i9eQ89WjdmUOdchbkkFD8DXR26HIFte8r57ze/5KVZq+mYm8lLo05iYCcN05LE42egBytfFFWHLhG4/rlZzFyxhRuGFHDLWV20g0USlp+Brg5darBhx14yM1JpmJ7GHcO7kZaSQu922fEuS6RW+blNRLtc5BCcc0z8ooRh90/l/imhYVon5jdRmEtSUIcuCePrrXu489V5fLRoA33zc7i4f168SxKJKT8DXbtcpJp3F3zDrS/NwQF3n9+DK07WMC1JPn4Gujp0qeScw8woaNGIkzo14+4LepLXVPNXJDn5uQitXS5JryIQ5K8fLeXWymFaBc0b8cRV/RXmktQ879D9/H0kx2bhmu38csJc5n+9nXN6apiWyD5+BrrW0JNSaXmAhz8oZuzUpeQ0TOevl/fl3N6t412WSJ3hZ6BrDT0p7dpbwQufr2LkCW2567zu5DTUMC2RqiJaszCz4Wa2yMyKzWz0YY7rb2YBM7soeiWGoQ49aezaW8G4aUsJBB3NGmUw5dbTuO8/+ijMRcKosUM3s1TgEWAYUALMNLNJzrmFYY77X+Cd2ij0AOrQk8K0xRv41cR5rNm2h15tsxlUkEuzRhqmJXIokXToA4Bi59wy51wZMB4YGea4m4EJwPoo1heedrkktK27y/j5P+Zy5ZOfk1EvhX9cfzKDCnLjXZZInRfJGnpbYHWV2yXAwKoHmFlb4EJgKND/UE9kZqOAUQD5+flHWuu3tMsloY16bjazV27hxjMKuHmohmmJRCqSQA/3djtX7fYDwB3OucDhLt/lnBsHjAMoLCys/hyR0xp6wlm/o5RGGWk0TE/j1yO6Uy/V6NlG81dEjkQkgV4CVB2K0Q5YU+2YQmB8ZZjnAiPMrMI591o0ijyIrimaMJxzvDK7hD+++SU/6NeO35zXgxPycuJdloiXIgn0mUAXM+sIfA1cAlxW9QDnXMd9H5vZ08AbtRbmoBdFE8Tqzbv59avzmL5kI/07NOHSgcewDCciNQe6c67CzG4itHslFXjSObfAzG6ofHxsLdd4MC25eO/t+d9w28tzMOAPI3vynwPbk6JhWiLHJKI3FjnnJgOTq90XNsidc1cde1k1FbRvyUWB7pt9w7S6tmzEKZ1z+d35PWjXRPNXRKLBz0VodejeKQ8EeeTDYn42fg4AnZo34rErCxXmIlHkZ6C7AGBwmB01UnfM/3obIx/+hHvfWUTAOfZWBOJdkkhC8nOWSzCg7twDpeUBHnx/CeOmLaNpZjp/u6If5/RsFe+yRBKWn4HuAlo/98DusgAvz1zN9/u25c4RPchuWC/eJYkkND8DXR16nbVzbwXPz1jJdYM70TQznSm3DaFppgZpicSCn4HugurQ66CPFq3nzlfns2bbHvq0y+HkgmYKc5EY8jPQgwFI8fP13ES0ZVcZ97y5kIlffE3nFo145YZB9GvfJN5liSQdPwNda+h1yvXPz+aLlVv46dDO3Di0Mxlp+t6IxIOfga419Lhbv72UzIw0MjPSuHNEd+qlptCjTeN4lyWS1Pxct1CHHjfOOV6euZozx0xlzJTFAPTJy1GYi9QBnnboQXXocbBqU2iY1sfFGxnQsSmXa5iWSJ3iZ6CrQ4+5t+ev5daX5pKaYvzxu724bEC+hmmJ1DF+Brp2ucTMvmFax7VqzJCuzfnt+T1ok9Mg3mWJSBh+pqI69FpXVhHkL+8v4afj5+Cco2NuJmOv6KcwF6nD/Ax07XKpVUUlW7ng4Y+5r/JFz7JAMM4ViUgk/FxycUFdfq4WlJYHuH/KYh6bvozmWRk8dmUhw3q0jHdZIhIhjwNdHXq07S4L8MrsEi7un8foc7uT3UDDtER84meg60XRqNlRWs5zM1Zy/WkFNM1M573bhtBE81dEvORnoOtF0aj44Kt13PnqfNZtL+XEvCacXNBMYS7iMT8DXS+KHpNNO/fyhzcW8vqcNXRt2YhHLx/EifkapiXiOz8DXR36Mfnx81/w79VbuOWsLvzk9M6kp2n5SiQR+Bno6tCP2DfbSsmqHxqmddd5PUhPS+G4VlnxLktEosjP1ky7XCLmnOPFz1cxrMowrd7tshXmIgnI3w49TS/e1WTlpl2MnjCPz5Zt4uROzbjy5PbxLklEapGfga419BpNnreW216eQ72UFP7ne725pH8eZhqmJZLI/Ax0raEf0r5hWt1bN2ZotxbcdV4PWmdr/opIMvB0DV0denVlFUEeeG8xN7347/3DtB69vJ/CXCSJ+BnousDFAeas3sr5f/mYB95bQlqKaZiWSJLyc8nFBTScC9hTFmDMlEU88fFyWmTV54kfFnJmdw3TEklWfga61tCB0HTEV/+9hksH5DP63G5k1dcwLZFkFlGba2bDzWyRmRWb2egwj19uZkWVfz41sz7RL7WKJF5D315azsMfLKEiEKRJZjrv3zaE/7qwt8JcRGru0M0sFXgEGAaUADPNbJJzbmGVw5YDQ5xzW8zsXGAcMLA2CgaStkN/b+E67nxtHht27KVf+6acXNCM7IYKchEJiWTJZQBQ7JxbBmBm44GRwP5Ad859WuX4GUC7aBZ5kCTr0Dft3Mvd/1zIP+euoVurLB67spDj2+XEuywRqWMiCfS2wOoqt0s4fPd9LfBWuAfMbBQwCiA/Pz/CEsNIsl0u+4Zp3TasKzcMKdAwLREJK5JAD/f2Qhf2QLMzCAX6qeEed86NI7QcQ2FhYdjniEgS7HJZu20PjevXIzMjjd+eHxqm1bWl5q+IyKFFkoolQF6V2+2ANdUPMrPjgceBkc65TdEp7xAS+JqiwaDj7/9aybAx07jv3dAwrV5tsxXmIlKjSDr0mUAXM+sIfA1cAlxW9QAzywcmAlc45xZHvcrqEvRF0eUbdzF6QhH/Wr6ZUzo346pBHeJdkoh4pMZAd85VmNlNwDtAKvCkc26Bmd1Q+fhY4LdAM+DRygFQFc65wlqrOgFfFH2zKDRMKz0thf/7/vH8oLCdhmmJyBGJ6I1FzrnJwORq942t8vGPgB9Ft7TDSKAXRfcN0+rZpjHDerTkrvN60LJx/XiXJSIe8nMhOgE69L0VAca8u4gbX/gC5xwdcjN5+LK+CnMROWp+BnowACl+lg7wxaotnPfQxzz0QTH101I1TEtEosLPWS6edui7yyr48zuLeerT5bRuXJ+nru7PGce1iHdZIpIg/Ax0T3e57C0P8s+iNVxxUnt+ObwbjTL8/M8vInWTn4niUYe+bU85z3y6gp+cXkCTzHTeu20I2Q00f0VEos+/QA9Wrjd70KG/s+Ab7nptPpt2lTGwY1MGdmqmMBeRWuNfoLtA6O863KFv2LGXuyct4M15a+neujFP/LA/vdtlx7ssEUlw/gV6sDLQ6/Aul5/8fTZzV2/j52d35fohBdRLrbu1ikji8C/Q62iH/vXWPWQ3qEejjDR+d35PMtJS6KL5KyISQ/61jvs79LoR6MGg49nPVnD2mKmMqTJMS2EuIrGmDv0YLN2wk9ETipi5YguDu+Ry9Skd4l2SiCQx/wK9juxyeaNoDbe9PJf6aSnce9HxXNRPw7REJL78C/T9HXp8Vov2DdPq3Tab4T1b8ZvzutMiS/NXRCT+tIYeodLyAPe+8xU/fj40TKt9s0weuvREhbmI1Bn+BXoc1tBnr9zMdx6aziMfLiUzI03DtESkTvJwyaUyTGOw5LJrbwX3vrOIZz5bQZvsBjxzzQCGdG1e619XRORo+BfoMVxyKQ8EmTxvLVee1J5faJiWiNRx/iXU/g69dgJ96+4ynvpkBTcP7UxOw3Teu30Ijetr/oqI1H3+BXotduhvzVvLXa8vYMvuMgYVNGNgp2YKcxHxhn+BXgvbFtdvL+W3ry/g7QXf0LNNY565pj8922iYloj4xb9Ar4UO/cYXvmBuyTbuGN6N6wZ3JE3DtETEQ/4FepS2LZZs2U1Ow3QaZaRx9wU9qV8vlYLmjaJQoIhIfPjXih5jhx4MOp7+ZDln3z+N+95dBEDPNtkKcxHxnocd+tHvcileHxqmNWvlFoZ0bc61p3aMcnEiIvHjX6Af5QUuJs1dw89fnkvDjFTG/EcfLjyxrYZpiUhC8S/Qj3ANPRh0pKQYfdplM6J3K+78Tg+aZ2XUYoEiIvGRsGvopeUB/vTWV9zw/Oz9w7QeuOREhbmIJCz/Aj2CDv3z5ZsZ8eB0xk5dSpOG6ZQHXIyKExGJH/+WXA7Toe/cW8H/vvUVz81YSV7TBjx/7UBO7ZIb4wJFROLDv0A/zC6XikCQdxd+wzWndOTn53SlYbp/pycicrT8S7xqu1y27CrjqU+W89Mzu5DTMJ33bz9dUxFFJClFtIZuZsPNbJGZFZvZ6DCPm5k9VPl4kZn1jX6plSrX0J2l8GbRWobdP5VHP1rKF6u2AijMRSRp1RjoZpYKPAKcC/QALjWzHtUOOxfoUvlnFPDXKNf5rfVfAfDUxLe48YUvaJ3dgEk3ncqAjk1r7UuKiPggknZ2AFDsnFsGYGbjgZHAwirHjASedc45YIaZ5ZhZa+fc2qhWu/pzmPonAC7fMIamg8Zx3ncGaZiWiAiRLbm0BVZXuV1Sed+RHoOZjTKzWWY2a8OGDUdaK6yYDsEKANJTgny3yXKFuYhIpUjSMNz746tv7I7kGJxz45xzhc65wubNj+LanB0GQ2oGWCqWmh66LSIiQGRLLiVAXpXb7YA1R3HMscsbAD+cFOrUOwwO3RYRESCyQJ8JdDGzjsDXwCXAZdWOmQTcVLm+PhDYFvX1833yBijIRUTCqDHQnXMVZnYT8A6QCjzpnFtgZjdUPj4WmAyMAIqB3cDVtVeyiIiEE9GmbefcZEKhXfW+sVU+dsCN0S1NRESOhLaIiIgkCAW6iEiCUKCLiCQIBbqISIKw0OuZcfjCZhuAlUf56bnAxiiW4wOdc3LQOSeHYznn9s65sO/MjFugHwszm+WcK4x3HbGkc04OOufkUFvnrCUXEZEEoUAXEUkQvgb6uHgXEAc65+Sgc04OtXLOXq6hi4jIwXzt0EVEpBoFuohIgqjTgV6nLk4dIxGc8+WV51pkZp+aWZ941BlNNZ1zleP6m1nAzC6KZX21IZJzNrPTzWyOmS0ws6mxrjHaIvjZzjazf5rZ3Mpz9npqq5k9aWbrzWz+IR6Pfn455+rkH0KjepcCnYB0YC7Qo9oxI4C3CF0x6STgX/GuOwbnPAhoUvnxuclwzlWO+4DQ1M+L4l13DL7POYSu25tfebtFvOuOwTn/Gvjfyo+bA5uB9HjXfgznfBrQF5h/iMejnl91uUPff3Fq51wZsO/i1FXtvzi1c24GkGNmrWNdaBTVeM7OuU+dc1sqb84gdHUon0XyfQa4GZgArI9lcbUkknO+DJjonFsF4Jzz/bwjOWcHZJmZAY0IBXpFbMuMHufcNELncChRz6+6HOhRuzi1R470fK4l9BveZzWes5m1BS4ExpIYIvk+dwWamNlHZjbbzK6MWXW1I5JzfhjoTujylfOAnznngrEpLy6inl8RXeAiTqJ2cWqPRHw+ZnYGoUA/tVYrqn2RnPMDwB3OuUCoefNeJOecBvQDzgQaAJ+Z2Qzn3OLaLq6WRHLO5wBzgKFAATDFzKY757bXcm3xEvX8qsuBXncuTh07EZ2PmR0PPA6c65zbFKPaaksk51wIjK8M81xghJlVOOdei0mF0Rfpz/ZG59wuYJeZTQP6AL4GeiTnfDXwJxdaYC42s+VAN+Dz2JQYc1HPr7q85LL/4tRmlk7o4tSTqh0zCbiy8tXik6jNi1PHRo3nbGb5wETgCo+7tapqPGfnXEfnXAfnXAfgFeAnHoc5RPaz/Tow2MzSzKwhoYuvfxnjOqMpknNeRehfJJhZS+A4YFlMq4ytqOdXne3QXRJenDrCc/4t0Ax4tLJjrXAeT6qL8JwTSiTn7Jz70szeBoqAIPC4cy7s9jcfRPh9vgd42szmEVqOuMM55+1YXTN7ETgdyDWzEuB3QD2ovfzSW/9FRBJEXV5yERGRI6BAFxFJEAp0EZEEoUAXEUkQCnQRkQShQBcRSRAKdBGRBPH/PTYVhcg/jr8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fit a model\n",
    "nb_model = nb_search.best_estimator_\n",
    "nb_model.fit(X,Y)\n",
    "# predict probabilities\n",
    "probs = nb_model.predict_proba(X_valid)\n",
    "# keep probabilities for the positive outcome only\n",
    "probs = probs[:, 1]\n",
    "# calculate AUC\n",
    "auc = roc_auc_score(y_valid, probs)\n",
    "print('AUC: %.3f' % auc)\n",
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_valid, probs)\n",
    "# plot no skill\n",
    "pyplot.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0f00cc08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAtJ0lEQVR4nO3dd3wVxd7H8c+cJEBCCBCkE0o0SC8mcCOgUqQICqEalaIgPRTFAoKCBUGlCUgJHRSRDtKkIyLSlaoUIRgCCSA1pJ7M88c55ElI4SApm+X3vq995ZzZ2Zab+73D7Oys0lojhBDCWCzZfQJCCCFSknAWQggDknAWQggDknAWQggDknAWQggDcs7sA9RXH8lwEJHClrgR2X0KwoCcnC3qYffxIJmzXX/y0MfLLNJyFkIIA8r0lrMQQmQlpQzbGH4gEs5CCFNRThLOQghhOCZpOEs4CyFMxiTpLOEshDAVk2SzhLMQwlzUw4/GMwQJZyGEuZik6SzhLIQwFYu0nIUQwoDMkc0SzkIIc5E+ZyGEMCCTdDlLOAshTMYk6SzhLIQwFYs8vi2EEAYkLWchhDAek2SzhLMQwlxkylAhhDAic2SzhLMQwlxknLMQQhiQhLMQQhiQ9DkLIYQRmeS11RLOQghTkZazEEIYkEmyWcJZCGEuckNQCCEMSMJZCCGMyCT9GhLOQghTMUk2SzgLIcxFRmsIIYQRyThnIYQwHovFHOks4SyEMBVljmw2yz8AhBDCTinHl/vuSr2llDqmlDqqlPpeKZVHKeWplNqklDpl/1kwSf0hSqnTSqm/lFJNk5T7KqWO2NdNVA50jEs4CyFMJaOyWSlVEugP+GmtqwBOQCAwGNiitfYBtti/o5SqZF9fGWgGTFFKOdl3NxXoAfjYl2b3uw4JZyGEqSiLcnhxgDPgqpRyBtyAMKAVMM++fh4QYP/cCliktY7RWp8FTgO1lVLFAQ+t9W6ttQbmJ9kmTRLOQghzyaCms9b6AjAGOA9cBG5orTcCRbXWF+11LgJF7JuUBP5JsotQe1lJ++d7y9Ml4SyEMBWLk3J4UUr1UErtT7L0uLsfe19yK6AcUALIq5TqmM6hU0t7nU55umS0hhDCXB7gIRStdTAQnMbq54GzWuvLtt2q5UAdIFwpVVxrfdHeZRFhrx8KeCXZvhS2bpBQ++d7y9Ml4ZwKD09Xxm15HQDPYu5YrZoblyMB6FU7mPg460MfY8K2N3B1z0XPWtMBeNK3BL3HNGVggzkPvW+ROapUrYyPT/nE75MnTaZkydT/derr58uB/Qce6ngffDCEffv34e6eD4tF8eGwD6lRo+ZD7fNRkIEPCJ4H/JVSbkAU0AjYD0QCXYDR9p+r7PVXAwuVUuOwtbR9gL1aa6tS6pZSyh/YA3QGJt3v4BLOqbj5bxRv1pwKwOvDGxB1O5Yfxu5KXO/kZMFqTXjo4xQokpfazXzYu+HUQ+9LZL7cufOwYvmKLD3mO4PepWnTpuzatYsRH49g5YpV99/oEZdRs9JprfcopZYCB4F44BC2VrY7sFgp1Q1bgLe31z+mlFoMHLfX76u1vtuS6w3MBVyB9fYlXRLODho8pzU3/43Cp2ZxTh4MI+pWbLLQnnOkL0Ne/I5LIddp/Fo12vT3xyWXE8f3hDKhzxoSElJ2MS36ahedhj2XIpwtFkWP0Y2pUb8cLrmdWPnNXn4M3o9SigGTW1D9ubJcOnsNZVGsn32QHcuOZ8nvQCQXGRlJUL8gbt68QXx8PP37D6BRw0bJ6ly+HMHbg97m9u1IrNZ4PvpoOH6+fuzatYvJ30wiNjYWL6/SjPxsJHnz5k3zWH5+fpw/fx6AuXPnsnzFMgDatW1H585duHPnDm8PeovwS+FYE6z07tWbF15onnkXb2QZOLWG1no4MPye4hhsrejU6o8ERqZSvh+o8iDHlnB+AF7lCzHo+bkkJGheH94g1TqlKzxGg5erElR3Jtb4BAZ+8yLPv1aNjQv+SFH3+O5/eKZ1RWrUL0fUrZjE8ubdniLyRgy9ak/HJZcTk3e9yb6Np3nStwTFyhaga9VvKFgkL/NO9GP97IOZdr0iuZiYaFq3aQ1AqVIlGT9uApMmTsLd3Z1r164R+EogDRs0TDbxzpq1a6lbtx69evbCarUSHR3NtWvXmDZ9KrNmzsbNzY2ZM2cwb95c+vTpm+axt23fho9PeY4dO8aKlctZ9P0PaK0JfOVl/GrVIvSfUIoULsK0qbZuslu3bmXuL8PALE7mGOcg4fwAti85lmoLOCnfRt6U9y3O9H09Acjl6sL1iMg06y/4bAedhj1H8PsbE8tqNXkC72pFea5dJQDy5s9DKZ9CVK1Xhu1LjqG15t/w2xzadjYDrko46t5ujbi4OCZMGM/+A/tRykJERDhXrlyhcOHCiXWqVqnC0GHDiI+Pp1HDRlSsWJF9+/Zy5swZXuv4WuJ+atSonuoxx4z9imnTp+HpWZDPPv2U3377jecbPY+bmxsAjZ9vzIEDB3imXj2+GvMlY8eO4bn69fHz9cvE34SxmWRSOgnnBxEdGZv42RqfkKxvK1ce+69SKX6a9zszPtjs0D4PbTtL108bUck/yc1cBRP7rWPfxtPJ6vq3KI8wjjVr1vDvtX9ZsngpLi4uPN+4EbGxscnq+PnVYsH8BezYsZ3BQ96n6xtd8fDIT52n6zBmzNj7HuNun/Ndu3fvTrVe2bLlWLJ4KT/v/JkJ48dTp06ddFvipmaSdDZH+z8bXDp3jfJPFQfAp2ZxipWzPV5/cMvfPNeuMgUK2/oP8xV0pWjp/Onu69uROwh8r17i930/naZV71o4Odv+6ynlU4g8bi4c+eU8z7WthFKKgkXyUqN+2Uy4MuGo27dv4elZCBcXF/bs2UNYWMrRURfCLuDp6Un79h1o26Ytx48fp3r16hw8dIiQkBAAoqKiOHfOsX8F+fn5sWXrFqKiorhz5w6bt2zG19eXiIgIXF1daflSS15/4w2OnziRodeak2TwE4LZRlrO/9GOZcdp0rkGMw/15s99Fwg9eRWAkBOXmTVsC2M2dkZZFPFxCXzddw3h52+kua89609x/fKdxO9rZx6kWNmCzDjYG6Xg+uU7DAtYyM/LjuPbyJs5R/sSevIqJ/Zc4PaNmDT3KzLXiy++RJ++vWnfoR0VKlTA29s7RZ19e/cxe84snJ1dcHNzY/So0Xh6evL5yM959913iI2ztbT79xtA2bLl7nvMSpUqE9CqNS8HdgBsNwQrVazEL7/8wpixX6GUBRdnZz766N57WI8OkzScUbZHvTNPffVR5h7gEeOaNxdRkbF4eLoydW9P+tWdyb/ht7P7tB7YlrgR2X0KwoCcnB++Oftm49kOZ87MTV0NG+XScs5hRq15DfcCeXDO5cSCT7fnyGAWIjNZnAybtw9EwjmHkScIhbgPk/RrSDgLIUzFJNks4fyw2vb358XuvqAUa2ccYOnXu3l9eANadPdNnI9jxgeb2bPe9hSgd9WiDJreEjeP3OgETa9a04mNiafbZ41o2rkG+Qrm4YV8KR4wEjnY0GFD2bFjO56enqxe9SMAbw96i7NnzwFw69ZN8uXzYMXyFfy45kdmz56duO3Jk3+xdMkyKlasmB2nniMZfRSGoyScH0K5ykV4sbuvbTKkWCtfbujE7rV/AbB0/O5k83GAbU6Ood+25fNOyzhzOBwPT9fESZR2//gXKybv4btTA7L8OkTmah0QwGuvvsrgIYMTy8aNHZ/4+YsvvyCfuzsAL734Ei+9+BIAJ0+eJKhfXwnmB2WSpvN9w1kpVQHbnKYlsc1BGgas1lo/ugMp7UpXLMzx30KJiYoD4Pcd53imdaU06/s1eZy/D4dz5nA4YJtg6a7je0LT2kzkcH5+tbhw4UKq67TW/PTTBmbPTnkvYe26tTRv3iKzT890TJLN6T+EopR6H1iEbSqRvcA+++fvlVKD09v2UXD2aDjVni2Dh6cruV1d8G9eniJeHgC0DqrNrD/68N6sANwL5AHAq/xjaK35ckNngg/0IvDdeuntXjwCDhzYT6FChShbpmyKdRs2rKdF80d08qKHoJwsDi9Gdr+WczegstY6Lmmhfb7SY9jmM03B/jaBHgA+tKAET2XAqRrP+T+v8P0XvzBmUxeibsdy5o9LWOMTWDV1L/M/3Y7W0PXThvQZ24wvu63EydlC1Xpl6FVrOtF34hi35XVOHgjj4Na/s/tSRDZJq3X8x+E/yJMnT7L5o4VjHomWM5CAbdLoexW3r0uV1jpYa+2ntfYzazDftW72QXr4TmPAc7O5+W8Uoaeuci0ikoQEjdaatTMOULG2bUL2y6E3+GPHOW5cvUNMVBy/rTuJj/0RcPHoiY+PZ/PmzbzQ7IUU69avWyddGv+RWR7fvl84DwS2KKXWK6WC7csGbK8DlztXkDiHRhGv/DzbpiJbvj+CZzH3xPX1Wlfk7FHbW2z2/nQa72pFye3qgpOThRrPlSXk+OVsOW+R/Xbv3k25cuUoVqxYsvKEhAR+2vgTzR/V+ZgfklLK4cXI0u3W0FpvUEqVB2pjuyGosL0Pa1+SGf4faZ8sC8SjkCvxcQlM6LuW29ej+WB+G56oURytNZfOXWdsz9UA3L4ezZJxvzJtX0/Qmt/WneK3dScB6PlFE55/tSq53VxY8s8g1s48yNyPt2XnpYkM8s47g9i7by/Xr1+nQcP6BPUNom3bdqxfn3rreP/+/RQtWhQvL69U9ibuy9iZ6zCZW0NkC5lbQ6QmI+bWGPDKIocz5+vvAw0b5TLOWQhhLgbvS3aUhLMQwlQM3pXsMGMP9DMYi0Ux42BvRv1oe73Q49WK8s2v3Zl9uC+fr34Nt3y5U2xTuJQH47e+wbzj/ZhzNIi2/f2TrW8d9D/m/9mfOUeD6PlFEwCq1CnNrD/6MG1vT0o+7gmAe/48fLmhcyZfoXgYO3fupHmLF2jarCkzZsxIsf7GjRv06x9EQOtWvPxyB06dOpm4buiwodR7pi4tW72UbJuxY8cQ0LoVg4e8n1i2evUqFiyYn3kXksOZ5YaghPMDaDvgaUJO/P/oindnBhA8eBNdq33DzhXHCXy3boptrPEJTBm0gS6VJtHHP5iAvrUpU9H2jrka9ctRr1UFulX7hjeqTOaHMbbHvTsMqsNHbRcx44PNtOxdC4BOHz7Hd5//nAVXKf4Lq9XKZyM/Zfq0YH5c/SPr1q3l9OnkrxkLnhFMhQoVWbliFaNGjebzUaMS17UOCCB4enCy+rdu3eLQ77+zcsUqrNYETp48SXR0NCtWriQw8JUsua4cyaIcXwxMwtlBhUt64N+iPGtnHkgs83qyEH/8fA6A/ZvO8GzblI9u/3vpNqcOXQQg6nYsIScu81hJ21OErXrXYuHoncTF2ga+XLdPlBQfZyW3qwt53FywxiVQwrsghUt6JB5LGM+RI4cp7VUaLy8vcuXKxQvNm7N129Zkdc6cOY3//2z/cvL29iYs7AJXrlwBbI94589fIFl9i8VCXFwcWmtiYqJxdnZm9uxZdOzYERcXlyy5rpxIKccXI5NwdlDQhBeY/t5P6CRv3z57NIK6LSsAUL99FYp4pf+uwGJlCuBTszgn7PNoeJUvRNVnyjDltx5M2N6VJ/1sz/ssHLWTd4Jb0m7g06yYvIc3Rz7PrA+3ZNKViYwQHh5BseL/P165WNGiRISHJ6vz5JMV2Lx5EwCHDx8mLCyM8HvqJJU3b16aNG5Mm7ZtKFmyFPnyuXP06FEaNWyUORdhEmZ5fNvYZ2cQT7coz7WISE4evJis/MuuKwnoW5vp+3vhli9XYgs4Na55c/HxskAmD1zPnVu29/45OVvIV9CVPv7BTHv3J0YsfhmA039cos/TM3ir4RyKexfkStgtlFJ8tKg9Qxe0pWCRvJl3seI/0aQyeuuepln3N7tz4+ZNWrdpzXcLv6VihYo4OTmlu99u3d5kxfIVvP/e+0ycNJGgfv1YunQJb739FtOmTc3ISzANs/Q5y2gNB1SpW5q6LZ/Ev7kPufI44+aRm6EL2jKy0zLebWq7MVPKpxD+LVKfB8HJ2cLHywLZ/N1hdq74/8n8LofeZOfy4wD8ue8CCQma/I+5cePK/7/stdOw5/j45cUMmNyCucO3UaxsAdr092fWMGlJG0mxokW5dPFS4vdL4eEUKVIkWR13d3c+H/k5YJuNrnGT5ylVqpRD+z9+wvZ3UrZMWUaN+pwF879l0Dtvcy7kXKqTJj3KlEmanCa5jMw144PNtPcaS2C58XwSuIRDW88ystOyxEe3lVJ0GvYcq6ftS3X792YFcP7EZZaM/zVZ+S8rT1Czoe2NzaV8CuGSyylZMDfrUoPf1p7k9vVo8ri5kJCgSUjQ5HGT/kajqVKlKiHnQwgNDSU2Npb169bRoEGDZHVu3rxJbKztbdtLly7Bz88Pd3f31HaXwqRJE+kX1J/4+HgSrLZpbSzKQnRUdMZeiAlIy1nQ6JWqBPStDcDO5SdYP+cQAIWK5+Pdma0Y3OJbqtYtTdPONThz+BIzD/UG/v/NKOtmH+L92QHMOdKXuFgro7osT9x3blcXmnapyTtN5gGweNxuPlkWSFyslU9fWZLFVyrux9nZmaFDh9G9x5skJCTQunUbfJ7wYdEPiwAIfDmQv/8+w+Ahg3FycuLxxx/n008+S9w+rUe8ATZv2UyVKlUTW+LVa9SgVUBLypd/kgoVKmT9xRqdwUPXUfL4tsgW8vi2SE1GPL79fp+VDmfOF1MCDJvk0nIWQpiK0UdhOErCWQhhKkbvS3aUhLMQwlRMks0SzkIIkzH4Y9mOknAWQpiKdGsIIYQBKScJZyGEMBxpOQshhAEZ/a3ajpJwFkKYizmyWebWEEKYS0bOraGUKqCUWqqU+lMpdUIp9bRSylMptUkpdcr+s2CS+kOUUqeVUn8ppZomKfdVSh2xr5uoHDi4hLMQwlSURTm8OOBrYIPWugJQHTgBDAa2aK19gC327yilKgGBQGWgGTBFKXV3TtipQA/Ax740u9+BJZyFEKaSUeGslPIAngVmAWitY7XW14FWwDx7tXlAgP1zK2CR1jpGa30WOA3UVkoVBzy01ru1bTKj+Um2SZOEsxDCVB6kW0Mp1UMptT/J0iPJrryBy8AcpdQhpdRMpVReoKjW+iKA/efdibtLAv8k2T7UXlbS/vne8nTJDUEhhKk8yEg6rXUwEJzGamfgKaCf1nqPUupr7F0YaR06tUOkU54uaTkLIUwlA1/wGgqEaq332L8vxRbW4fauCuw/I5LU90qyfSkgzF5eKpXydEk4CyFMJaNGa2itLwH/KKWetBc1Ao4Dq4Eu9rIuwCr759VAoFIqt1KqHLYbf3vtXR+3lFL+9lEanZNskybp1hBCmIolYx9C6Qd8p5TKBfwNvIGtUbtYKdUNOA+0B9BaH1NKLcYW4PFAX6313bc+9wbmAq7AevuSLglnIYSpZOTT21rr3wG/VFY1SqP+SGBkKuX7gSoPcmwJZyGEqcjcGkIIYUAmyWYJZyGEuSiTTK4h4SyEMBVpOQshhAFl8GiNbCPhLIQwFWk5CyGEEZkknSWchRCmYpJslnAWQpiLjHMWQggDMkk2SzgLIcxFRmsIIYQBmSOaJZyFECYjfc5CCGFAJslmCWchhLlIy1kIIQxIbggKIYQBmaThLOEshDAXCWchhDAg6XMWQggDMkk2Z344b4kbkdmHEDlQY9dPsvsUhAFtzYC8kJazEEIYkJLRGkIIYTzSchZCCAMySTZLOAshzEVazkIIYUAmyWYJZyGEuUjLWQghDEjm1hBCCAOSlrMQQhiQjHMWQggDMknDWcJZCGEu0q0hhBAGJDcEhRDCgKTlLIQQBmSSbJZwFkKYjEnSWcJZCGEqZunWsGT3CQghREZSyvHFsf0pJ6XUIaXUGvt3T6XUJqXUKfvPgknqDlFKnVZK/aWUapqk3FcpdcS+bqJy4P9BJJyFEKZicVIOLw4aAJxI8n0wsEVr7QNssX9HKVUJCAQqA82AKUopJ/s2U4EegI99aXbf63D07IQQIidQSjm8OLCvUkALYGaS4lbAPPvneUBAkvJFWusYrfVZ4DRQWylVHPDQWu/WWmtgfpJt0iThLIQwlQcJZ6VUD6XU/iRLj3t2NwF4D0hIUlZUa30RwP6ziL28JPBPknqh9rKS9s/3lqdLbggKIUzlQe4Haq2DgeDU96NeBCK01geUUvUdOXRqh0inPF0SzkIIU8nA0Rp1gZZKqeZAHsBDKfUtEK6UKq61vmjvsoiw1w8FvJJsXwoIs5eXSqU8XdKtIYQwlYzqc9ZaD9Fal9Jal8V2o2+r1rojsBroYq/WBVhl/7waCFRK5VZKlcN242+vvevjllLK3z5Ko3OSbdIkLWchhKlkwdwao4HFSqluwHmgPYDW+phSajFwHIgH+mqtrfZtegNzAVdgvX1Jl4SzEMJUMuMhFK31dmC7/fNVoFEa9UYCI1Mp3w9UeZBjSjgLIUzFJA8ISjgLIcxF3oQihBAGJC1nIYQwILNMfCThLIQwFQlnIYQwIJNks4SzEMJcpOUshBAGZJJslnAWQpiLtJyFEMKAsuDx7Swh4SyEMBWTNJwlnIUQ5iLhLIQQBqRSnds+55FwvkeVqpXx8Smf+H3ypMmULJn6G2V8/Xw5sP/AQx3vgw+G8OvuX9n40yZy5crFtWvXaN+hHZs3bXmo/YrM4eHpypiNtql8PYu6k2BN4PqVOwD0eXoG8XHW9DZ3yLjNr1OomDuxMfFE3Y7lq+6r+Ofk1Yfe76NCWs4mlTt3HlYsX5Glx7RYnFi+fBmBga9k6XHFg7v5bxQ9/KYB0OXD+kTdjmXx+F8T11ucLCRYE9La3GEjuyzn5IEwWrzpS8/RTRjW5vuH3uejQm4IPiIiIyMJ6hfEzZs3iI+Pp3//ATRqmHwq18uXI3h70Nvcvh2J1RrPRx8Nx8/Xj127djH5m0nExsbi5VWakZ+NJG/evCmO0blTZ+bNn0e7du1TrJs1exY/bdhAbFwsjRo9T7+gfgBMnTqFNWvXUKxYMQoWKEilypXp+kbXzPkliHS9NyuAW/9G8USNYpw6dJGoW7HJQnvWoT58ELCQ8JDrPP9qNdoE/Q/nXE6c2BvK10FrSUhI+3Vyh3eG0LafPwA9RzemdlMfNJpvP/+Z7UuO4VnMnY8WtsfNIzdOThYmBK3hyK7zWXLdRiVD6UwqJiaa1m1aA1CqVEnGj5vApImTcHd359q1awS+EkjDBg2T/QGsWbuWunXr0atnL6xWK9HR0Vy7do1p06cya+Zs3NzcmDlzBvPmzaVPn74pjlm8eHGeesqX1T+upkH9Bonlu3bt4nxICD/8sBitNX2D+rB//z7y5HFl46ZNLFu6HKvVStt2bahUuXLm/3JEmkr5FOLdpvNJSNB0+bB+qnVKV3iMBu0r0+/ZWVjjExgwqQWNXq3Gpm//SHO/dVqU5+zRcJ5pXZHHqxeju+9U8j/mxpTdPTi8M4RGgVXZt/E0343eicWiyO3mkklXmHOYJJslnO91b7dGXFwcEyaMZ/+B/ShlISIinCtXrlC4cOHEOlWrVGHosGHEx8fTqGEjKlasyL59ezlz5gyvdXwtcT81alRP87g9uvegb1Bfnnv2ucSyXb/uYtevu2jTtg0Ad+7cISQkhMjISBo2bEiePHkAkgW6yB47lh1LtwUM8FRDb3yeKsHU33oAkDuPM9cjIlOtO3ReG2Ki47l07jqTBq6j/cCn2frDURISNNciIjm88xwV/Ery1/4w3p3RCmcXJ35Z/Sdn/riU4deW00jL+RGxZs0a/r32L0sWL8XFxYXnGzciNjY2WR0/v1osmL+AHTu2M3jI+3R9oyseHvmp83QdxowZ69BxypQpQ8UKFdiwYUNimdaa7t178HKHl5PVnTdv7kNfl8hY0ZFxiZ+t8QnJJnzPlcf2PzOlYOOC35k57P43e+/2Od+VVuAc/iWEgQ3n4N/chyFzW/PD2F/TbYk/EsyRzfL27fu5ffsWnp6FcHFxYc+ePYSFpXyj+YWwC3h6etK+fQfatmnL8ePHqV69OgcPHSIkJASAqKgozp07m+6xevTsyZy5sxO/16tbj+XLlxMZaWtdhYeHc/XqVZ56ypft27cRExNDZGQkO37ekYFXLB7WpZDr+NQsDoBPzeIUK1cAgINbz/Jsm0oUKGy775CvoCtFS+d3aJ+Hd4bQoH1lLBZF/sfcqFavDCf2XaBo6fxci4hk7ayDrJ9ziPL24z7KMurt29lNWs738eKLL9Gnb2/ad2hHhQoV8Pb2TlFn3959zJ4zC2dnF9zc3Bg9ajSenp58PvJz3n33HWLjbC3t/v0GULZsuTSP5fOED5UqVeL48eMA1K1bl7//PsOrr9lGcbi5ufHF6C+pWrUqDRo0pHWbAEqUKEHlylXI5+6eCVcv/ouflx+nScfqBO/vxV/7LxBqHwYXcuIys4dv5cv1nVAWhTXOytf91xF+/sZ997lz5Qkq+ZdixoHeaDTTh2ziWvhtmnSqzstv1yU+3krU7VhGv5G1I42MyCyjNZTW6feTPSxr/H064sR/EhkZSd68eYmKiqJzl058POJjKlXKOTcFG7t+kt2nIAxoa9yIh07WI0cuOZw5VasWM2ySS8s5hxoxYjinz5whNjaGVq0CclQwC5GZjN5d4SgJ5xzqq6/GZPcpCGFIJslmCeeMcvHiRYYMGcyVq1dQStGhfQc6derMxIlfs3XbVpSyUKiQJ5+PHEWRIkX49dddjBs/jri4OFxcXHhn0Lv4+/tn92WIDNKm3/9o0dUXpWDt7IMsm/hb4roOb9Wh15dNCCj2JTev3qFCrZK8PfUlwBYs8z7Zzi+r/gSgfvvKvDbkWZwsit/WnyJ4yKZsuZ6cRFrOIhlnZyfee+89KlWqTGRkJO3at+Xpp+vQtWs3+vcfAMCCbxcwZeoURgwfQYGCBZnyzVSKFCnCqVMn6d6jO9u3yagLMyhbuQgtuvrSp84M4mKtfLG2I7+tO8mF0/9SuJQHvs97Ex5yPbH+2aMR9PpfMAnWBDyLuTPjQG9+XXMS9/y56Tm6Cb3+N50bV+7w/uwAajYox6Ft6Y/6edSZJJtlKF1GKVy4SGK/b968efH2fpyIiHDck4yiiIqKSvzDqVSxEkWKFAHgiSd8iImJSTF+WuRMZSo8xvG9ocRExZFgTeCPn89Rr1VFAPqMacb0IZtIeh/+bj2wjYm+e5O+uHdBQk9d5YZ9YqWDW/7m2TaVsvZiciCLRTm8GJm0nDPBhQsXOHHiBNWq2Z4InPD1BFavXoW7uztz58xLUX/jxo1UrFiRXLlyZfWpikxw9lgEXT9phIenKzFR8fzvBR9OHgijzotPciXsJn8fDk+xTYXaJXkvuBVFyxRg1OvLSbAmcOH0v5R+8jGKlinA5dCb1G1ZAZdcTtlwRTnLI99yVkq9kc66Hkqp/Uqp/TNmBP/XQ+RIkZGRDBjYnyGDBye2mgcOGMjWLdt48cWX+G7hd8nqnzp9inHjxzJi+MfZcboiE5z/8wqLxvzCVxs688Xajpw5HI41PoHXhjzD3BHbUt3mz70X6FpjCr2fDubV95/BJbczt69HMyFoDR8tbMfX29/gUsh1rPEPP+Od2ZnlIZT/PM5ZKXVea136fvUepXHOcXFx9OnTm7p16/H666+nWH8h7AK9e/di9aofAbh06RJvdH2dkZ99zlNPPZXFZ5u9HqVxzt0+bcS1iNu8NvhZYu7YHvMuXMqDK2G36FNnBtfCbyerP3ZTF6YP3pTs8W2AFm/6UvJxT1PfFMyIcc6nTl1xOHN8fB4zbEKn262hlDqc1iqgaMafTs6ltebDj4bh7e2dLJjPhZyjbJmyAGzbtg3vcrYnDG/evEnv3r14a+Dbj1wwPwoKFM7L9cuRFPHKzzMBFQl6ZibLJ+1JXL/w1EB6+Qdz8+odipUtQMQ/N0mwJlC0dH68yj/GpXPXk+3HvUAeWvWqxSevLMmmK8o5jN4idtT9+pyLAk2Ba/eUK+DXlNUfXQcPHmT16tWUL18+ccrRgQMHsnzZMs6eO4vFYqFE8RIMHz4CgIULv+P8P+eZOm0qU6dNBWDmjJkUKlQouy5BZKARizvg4emGNd7K1/3Xcvt6dJp1q9YtzSvv1iM+PgGdoPm631puXrXdBAwa1wzvasUAWDByB6Gn5I0o92OSbE6/W0MpNQuYo7X+JZV1C7XWr97vAI9St4Zw3KPUrSEclxHdGn//fdXhzPH2LmTYKE+35ay17pbOuvsGsxBCZDV5wasQQhiRObJZwlkIYS5m6XOWJwQf0NBhQ6n3TF1atnop1fU/rvmRgNatCGjdildfe4U///wzcd3OnTtp3uIFmjZryowZMxLLx44dQ0DrVgwe8n5i2erVq1iwYH7mXYjIEBaLYvq+noxcaevle7x6MSb/8ibB+3sx9bceVKhVMtXt8ubPw/BFHZh7JIg5h/tSyb8UAG+MaMCMg70J3t+LL9d1olDxfABUruPFjIO9mbK7OyUe90zcxxdrO2bBVeYs6gH+Y2QSzg+odUAAwdPTfrCmVMlSzJs7n5UrVtGrV2+GjxgOgNVq5bORnzJ9WjA/rv6RdevWcvr0aW7dusWh339n5YpVWK0JnDx5kujoaFasXElg4CtZdVniP2rT35/zJ64kfu85qjHzP91OD79pzB2xjR6jGqe6XdD4ZuzbeJrXq06mu+80Quz7+GHsr3R/aio9/Kaxe91JOg2zvVOyw8A6jOjwA7OGbaFlTz8AOg19lu9G78zU68uJlHJ8SX8/yksptU0pdUIpdUwpNcBe7qmU2qSUOmX/WTDJNkOUUqeVUn8ppZomKfdVSh2xr5uoHBjvJ+H8gPz8apE/f4E019esWZP8+W2vHqperTrh4bYXbh45cpjSXqXx8vIiV65cvNC8OVu3bcVisRAXF4fWmpiYaJydnZk9exYdO3bExUXepGxkj5X0wP8FH9bNPphYprXGzSM3AHnz5+Zq2K0U27nly021emUSt4uPsxJ5wzbU7s6tmMR6edxcuDsJR3ycldyuLuR2c8Eal0AJ74I8VsKDwztDMu36cqoMfEIwHhikta4I+AN9lVKVgMHAFq21D7DF/h37ukCgMtAMmKKUuvu8/VSgB+BjX5rd7+DS55yJli1fxjPPPANAeHgExYoXS1xXrGhRDh8+TN68eWnSuDFt2rbB39+ffPncOXr0KH369M2u0xYO6jvWNomRm3vuxLJvBm3gi7Wd6PVFEywWRb9nZ6XYrrh3QW5cucN7swJ4vFpRTh68yDdvrSfa/vRg108a0qRjdSJvxPB247kALPzyF96e+hIxUXGMen0Fvb5owpwRW7PkOnOcDOqt0FpfBC7aP99SSp0ASgKtgPr2avOA7cD79vJFWusY4KxS6jRQWyl1DvDQWu8GUErNBwKA9ekdX1rOmWTPnj0sX76MQW8PAkCTytBL+/9zd+v2JiuWr+D9995n4qSJBPXrx9KlS3jr7beYZn9ARRiLf/PyXL8cyamDF5OVt+xZiynvbCDQezzfvPMT7wS3SrGtk7MFn5rFWT19Hz1rTSc6MpZX3quXuH72R1sJ9B7P5u8PE9CnNgBn/rhEUL2ZDGo8jxLlCnL14i2UUnz4XTuGzGtDwSJ5M/eCcxD1AIvD+1SqLFAT2AMUtQf33QAvYq9WEvgnyWah9rKS9s/3lqdLwjkT/PXXX3w0/EMmT5pMgQK27qhiRYty6eKlxDqXwsMTpwy96/gJ24tdy5Ypy6rVqxg/bjynTp/iXMi5LDt34Zgqdbyo8+KTLDw1kA+/a0fNBuUYMq8NTTpVZ+eKEwDsWHos1RuCl0Nvcjn0Jn/uvQDAz8uOJ76tO6mti47wbOuUU4R2/OBZFozcQecPn2PuJ9vZ/N1hWgf9L4OvMOd6kG6NpJO02ZceqezPHVgGDNRa30zv0KmU6XTK0yXhnMHCwsLoP6A/o0d9kexN21WqVCXkfAihoaHExsayft06GjRokGzbSZMm0i+oP/Hx8Ynz+1qUheiotB/9Fdlj5rAtvFxuHK/6TODT15ZyaNtZRnVZztWwW1R/tiwANRuU48LplI9bXwu/TUToDbzK2x7Vf6qhNyEnLgNQ8gnPxHp1XnqS839dSbZt0841+G39SW5fjyaPqws6QaMTtK1/WjwwrXWw1tovyZLsbr9SygVbMH+ntV5uLw5XShW3ry8ORNjLQwGvJJuXAsLs5aVSKU+X9Dk/oHfeGcTefXu5fv06DRrWJ6hvEHHx8QAEvhzI1GlTuHHjOp98ans82dnZiSWLl+Ls7MzQocPo3uNNEhISaN26DT5P+CTud/OWzVSpUjWxNV29Rg1aBbSkfPknqVChQtZfqPhPxvb+kaBxzXBythAbHc/Y3rYZCAsVz8c701sypKVtythJA9fzwfy2OOdy4uLf1/jyzZUAdB/5PF7lHyNBayJCrjO+75rEfed2daFJp+q898ICAJZM2M2IxR2Ij7XyWcdlWXuhBpZRk+jbR1TMAk5orcclWbUa6AKMtv9claR8oVJqHFAC242/vVprq1LqllLKH1u3SGdg0n2P/1+nDHWUzK0hUiNza4jUZMTcGhfDbjqcOcVLeKR5PKVUPWAncAS4O5H2B9gCdjFQGjgPtNda/2vfZijQFdtIj4Fa6/X2cj9gLuCK7UZgP32f8JWWsxDCVDLqCUH7hG9p7a1RGtuMBEamUr4fqPIgx5dwFkKYitGf/HOUhLMQwlzMkc0SzkIIczH4S7UdJuEshDAXk0xLJ+EshDAVc0SzhLMQwmRM0nCWcBZCmIxJ0lnCWQhhKuaIZglnIYTJODBPc44g4SyEMBWTZLPMSieEEEYkLWchhKmYpeUs4SyEMBlzpLOEsxDCVKTlLIQQRiThLIQQxmOWKUNltIYQQhiQtJyFEKZilj5naTkLIYQBSctZCGEq8vi2EEIYkTmyWcJZCGEuJslmCWchhMmYpFtDbggKIYQBSctZCGEq5mg3SzgLIUxGRmsIIYQRmSObJZyFEOZikmyWcBZCmIxJ0lnCWQhhMuZIZwlnIYSpmCOaJZyFECZjksEaEs5CCJMxSTrLE4JCCGFA0nIWQpiKSRrO0nIWQggjkpazEMJUzPL4ttJaZ/c5PDKUUj201sHZfR7CWOTvQqRGujWyVo/sPgFhSPJ3IVKQcBZCCAOScBZCCAOScM5a0q8oUiN/FyIFuSEohBAGJC1nIYQwIAlnIYQwIAnnLKKUaqaU+kspdVopNTi7z0dkP6XUbKVUhFLqaHafizAeCecsoJRyAr4BXgAqAa8opSpl71kJA5gLNMvukxDGJOGcNWoDp7XWf2utY4FFQKtsPieRzbTWPwP/Zvd5CGOScM4aJYF/knwPtZcJIUSqJJyzRmozscgYRiFEmiScs0Yo4JXkeykgLJvORQiRA0g4Z419gI9SqpxSKhcQCKzO5nMSQhiYhHMW0FrHA0HAT8AJYLHW+lj2npXIbkqp74HdwJNKqVClVLfsPidhHPL4thBCGJC0nIUQwoAknIUQwoAknIUQwoAknIUQwoAknIUQwoAknIUQwoAknIUQwoD+D95tOB5kIE55AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nb_model = nb_search.best_estimator_\n",
    "nb_model.fit(X,Y)\n",
    "\n",
    "nb_predicted = nb_model.predict(X_valid)\n",
    "nb_matrix = confusion_matrix(y_valid, nb_predicted) \n",
    "# print(lr_matrix)\n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in nb_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in nb_matrix.flatten()/np.sum(nb_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(nb_matrix, annot=labels, fmt='', cmap='Purples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "42d5c8ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.98      0.98      0.98      9738\n",
      "         1.0       0.98      0.98      0.98      9671\n",
      "\n",
      "    accuracy                           0.98     19409\n",
      "   macro avg       0.98      0.98      0.98     19409\n",
      "weighted avg       0.98      0.98      0.98     19409\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "nb_report = classification_report(y_valid, nb_predicted) \n",
    "print(nb_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f38ad213",
   "metadata": {},
   "source": [
    "### LSVC Further Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "c3b895ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('accuracy', 0.98436, 0.0009)\n",
      "('neg_log_loss', nan, nan)\n",
      "('roc_auc', 0.99624, 0.00041)\n"
     ]
    }
   ],
   "source": [
    "print(scoring_cv('accuracy',lsvc_search.best_estimator_))\n",
    "print(scoring_cv('neg_log_loss',lsvc_search.best_estimator_))\n",
    "print(scoring_cv('roc_auc',lsvc_search.best_estimator_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d9d35ba0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAD4CAYAAAAw/yevAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAs0klEQVR4nO3dd3gUxRvA8e9cEiQhlIROEnrvXZAOIp0gCFKkKBIFBASVJr1IUxSk+KNKR6T3Ir33Ip1Ib0lokoSQOr8/7oiJKRySsjnez/Psk7vZ2Z0dhTfDu7OzSmuNEEIIYzEl9wUIIYSISYKzEEIYkARnIYQwIAnOQghhQBKchRDCgOwTu4HPVTqZDiJi+CXwVnJfgjAip/TqdU/xKjHnF/30tdtLLDJyFkIIA0r0kbMQQiQlWxlxSnAWQtgUe2XYTMUrkeAshLApJtuIzRKchRC2RdIaQghhQCZJawghhPHIyFkIIQxIcs5CCGFAdpLWEEII45G0hhBCGJCkNYQQwoBk5CyEEAYkU+mEEMKA7G0jNktwFkLYFklrCCGEAZmwjaGzBGchhE2R2RpCCGFAktYQQggDkpGzEEIYkCy2L4QQBiRpDSGEMCBJawghhAHJVDohhDAgGTkLIYQB2UlwFkII45G0hhBCGJCkNYQQwoBkKp0QQhiQjQycJTgLIWyLLLYvhBAGJGkNIYQwINsYN9vOLxkhhABAKWX1ZsW5eiulzimlziqlliilUiulXJVS25RSVyw/XaLUH6CU8lZKXVJK1YtSXk4p9adl32RlReMSnIUQNkW9whbveZRyA3oC5bXWxQE7oDXQH9iutS4AbLd8RylV1LK/GFAfmKaUsrOcbjrgBRSwbPVf1g8JzkIIm2J6hc0K9oCjUsoecALuAp7APMv+eUAzy2dPYKnWOlhrfQ3wBioqpbID6bTWB7XWGpgf5Zh4+yGEEDZDKeu3+Git7wDfAzeBe8DfWuutQFat9T1LnXtAFsshbsCtKKe4bSlzs3z+d3m8JDgLIWyKCWX1ppTyUkodi7J5vTiPJZfsCeQBcgBplFIfxdN0bOFex1MeL5mtIYSwKa8yW0NrPQOYEcfud4FrWms/AKXUSuAdwEcplV1rfc+SsvC11L8NeEQ53h1zGuS25fO/y+MlwTkWaVxd+XL7WgDSZcuKDg/H3+8BAGMr1iI8NPS12+izcwNvOadhTIWaAOQsV4YPvh/FxFqNXvvcInEUKVeJgvnzRX6f+uME3HPkiLVumXdqcPLA7tdqr/+Q4Rw5foK0zs6YTCaG9P+GMqVKvtY53wQJuLbGTaCSUsoJCALqAMeAQKAjMNbyc42l/lpgsVJqIuaRdgHgiNY6XCnlr5SqBBwGOgA/v6xxCc6xCHz0iNFlqgLQeOgAggMC2PbDP/8tTXZ2RISHv3Y7abNkplj9upzbvO21zyUSX+q33mLNb4uStM2+X/akft067Dt4iCGjx7Ju2eIkbT8lUgk001lrfVgptRw4AYQBJzGPsp2BZUqpzpgDeEtL/XNKqWXAeUv97lrrF4GiK/Ar4AhssmzxkuBspY5zpxP46DEeZUpy88Rpgv0DogXtwX8eYlrjVjy8cZOK7T6kds/PsUvlwLXDx1jSrQ86IiLGObdOmEzDQd/ECM7KZOL9scMpWLMa9m+lYvfUmeydMRelFK2n/ECBGlV4cO0GJpOJA3MWcGLFmhjnFokv8NkzuvX+mqdP/QkLC6NXt895t1aNaHV8/R7Qu99AAgIDCQ8PZ9jAfpQvW4Z9Bw/x8/QZhISG4uHuxpjhQ0jj5BRnWxXKluHmLfO9prkLFrFizToAPnjfk07t2vAsKIgv+w7kvo8PERERdOvSmYb16iZe5w0sIR9C0VoPBYb+qzgY8yg6tvqjgdGxlB8Dir9K2xKcX0GWgvn56d2m6IgIGg8dEGudbIULUv7D5oyvUpeIsDDaTJ1IxXYfcnjBkhh1rx48Qun3G1OwZjWe+wdEllfp3IGgv58ytmJN7FOl4pv9Wzm/dQe5ypUmY+6cjCxRibRZMjP0wlEOzFmQaP0V0T0PDsbzw3YAuLvlYNL4MUz9YTzOzs48evyEDzt+Qp2a1aM93LB+0xaqvlOJrp9+Qnh4OEHPn/Po8ROmz5zD3P9NxcnRkRlz5zF3wWK++OzTONvesWcvBfPn5+z5C6xcu55lC+aitaZV+4+pWK4st27fIUvmTMz4+UcA/KP8eXrTyJKhb6ATv6+OdQQcVeE6NclZrjQDju4CwMHREX9fvzjrbxo1gYaDvmFlv39+ORd9rzZuJYtT9gNPABzTpyNLgXzkq1qZ47+vQmvNUx9fLu/c+/qdElb7d1ojNDSMiVOmc/TESUxK4ePrx4OHD8mcKVNknRLFijBw+CjCwsJ4t1ZNihQqyM7je/G+do02nT6NPE/pkrEPqsb/NJnps+bg6uLC6KGDOHjkKO/WqomToyMAdWvX4tiJU1SrUolxP05iwqSfqVWtKuXLlkm0/w5GJ4vtv4FCAgMjP4eHhaFM/8xEdEid2vxBKQ7NW8zqgcOtOuelnXtoOnIQeStV+KdQKX7r8Q3nt26PVrdEo3oI41i3aTOPHj9m5aL5ODjYU7uhJ8EhIdHqVChXloWz/sfuffvpO2gonTt8RLp06ajy9ttMHDvqpW28yDm/cODwkVjr5cmVi5WL57N73wF++HkaVSq9He9I3JbZRmiWec7/2cPrN/EoWxoAjzKlyJQnFwCXtu+i7AfNSJvZPHpycnHBNadHXKcBYNPo73mvb6/I7+e3bKd6186Y7M2/O7MUyE8qJye89x2kTAtPlFKkzZKZgjWrJULPhLX8AwLI6OKCg4M9h44e4869ezHq3Ll7j4yuLrRq3owWzZpy7uIlSpcozonTp7lx05xDDgp6zrUbN6xqs0LZMvyxazdBQc95FhTEHzt3Ub5saXx8/XBMnRrPRg3o3KEd5y9eTNC+piQJ9RBKcpOR8390csUaKnVow7cn93Hj6Al8LnsDcO/CJdYMGknPratRJhPhoaEs7f41j27eivNcZzdtjZyqB7B/1jwy5s7Jtyf2glIE+D1gerO2nFyxhsJ1ajDk7GF8Lntz7fAxgv5+muh9FbFr0qA+XXv1oXnbDhQpVJC8eXLHqHPk2HFmz1+Ivb09Tk6OjBs5DFdXF8YMH0KfAYMIsUzL/LLb5+TJleulbRYrUpjmTRrRsn0nwHxDsGjhQuw9cJDxP/2MSSns7e0ZNrBfQnY1RTF4zLWaMj/qnXg+V+kSt4E3zFtp0hAcGEgaV1f6H9nJhCp1eerj+/IDDeaXwLh/WYk3mFP6146tu7K4Wx1zavreNmwsl5FzCtN9/TIcM6THPlUqNo4clyIDsxCJyc6w4fbVSHBOYeQJQiHiZyOxWYKzEMK2JNQTgslNgvNrqt2zK1W6dEQpxb6Z89gxaRqNhw6gapeOkTf51gwcwdlNWwFwK1GMdv+bROp0adEREYypUJOw4GA8Rw3m7Q5tcHLJwJdpY1+vQaRMA4aNZNeefWR0dWH98qUAfNlvINeum2do+PsHkDatM2t+W8TajZuZPe+fB4suXfFm1ZIFFClUMFmuPSUy+iwMa0lwfg05ihWhSpeO5sWQQkLosXklZzdsAWD7j1OjrccB5jU5Pl44k7ntvbhz5ixpXF0jF1E6s24zO6fMYMSVk0neD5G4mjdpxEcftqTf4GGRZT+N+y7y89gffsLZ2RmApg3r07Sh+SUZl65406331xKYX5GtzA9+aXBWShXGvKapG+Y1SO8Ca7XWFxL52gwvW5FCXDt0lNCgIACu7N5P6fcbx1m/6Ht1uHPmHHfOnAXMCyy9cO3w0cS9WJFsKpQry+27sa8QqbVm07Y/mPe/aTH2bdi8lcb130vsy7M5NjJwjv+XjFKqH7AUc3+PAEctn5copfon/uUZ292z5ylQvQppXF1xcHSkeMP3cPEwL9ta8wsvBp0+QPvZU3HKkAEwr82htabH5lUMPL6H977pFc/ZxZvg2ImTZHR1JXeunDH2bdy6jUb15anQV2VSyurNyF42cu4MFNNaR1vA2LJe6TnM65nGYHmbgBdANd6iKKkS4FKN5/7Fy2wZ9yO9tq0mOCCQ26f/JCIsjN3TZ7Fh5DjQmqYjB9Hih9Es6NwdO3s78letxJgKNQl5FkTv7eu4cfwUl3a83rq/IuVav3krjWMJwKf/PItj6tTR1o8W1jF2yLXey9IzEZgXjf637JZ9sdJaz9Bal9dal7fVwPzCgTkL+K5cdX6o0YDAR4/xvfIX/r5+6IgItNbsmzmP3BXLAfD49l2u7N5P4MNHhAYFcXbjVnKWLZXMPRDJJSwsjG07dtGw3rsx9m3YspVGktL4T5RSVm9G9rLg/CWwXSm1SSk1w7Jtxvw6cPk3OUSuoeHi4U6Z5k05umQ56bJljdxf+v0m3D1rTs+f37Idt5LFcHB0xGRnR4EaVbh3/lKyXLdIfgcOHyVv7lxky5o1WnlERASbt+2gUT0Jzv+FSVm/GVm8aQ2t9WalVEGgIuYbggrz+7CORlnh/43mtWIhzhnNsy6WdP+KZ0+e0Gn+DDxKl0BrzcPrN1n0mfn32LMnT/hj4lQGHN2F1ppzG7dydqN5dkfzcSOo0LYlqZycGHPrAvtnzWf98DHJ2TWRQPr0H8SR48d5/OQJ1es1psfnXWj5vicb4xgdHz1xkmxZs+Dh/tIXNItYKKNHXSvJ2hoiWcjaGiJWCbC2xplcua2OOSVvXDdsJJd5zkIIm2L0XLK1JDgLIWyKjcRmm3mYJkkok4mBJ/bSbd0yANxKFqfvgT8YfOYg3db+Ruq0aWMc4+LuRu8d6xl6/ihDzh6mds+u0fbX/OIzhl08zpCzh2k+bgQA+d55m0GnD9D/yC4y58sLgGP69PTYvCqReyhex579B6nX7APqNm3OjDnzYuz/++lTuvf5hiat2vLBR5247P1X5L4Bw0ZSuXY9Gn/QOtoxEyb9TJNWbek76J/XmK1ev5F5i5cmXkdSuDdltoaIonavrty/cDnye/tZU1jVfygjS1bm1Kp11I3loZLwsDCWf/Utw4tWYFylOtTo3oXsRQoBULBmNUp5NmRUycqMKP42276fDMC7X/VgRov2rBk4nOpdOwPQcHBfNn/3fRL0UvwX4eHhjBg7nllTJrFhxW+s37wF77+uRqvzy+xfKVKoIOuWLWbcyGGMnvBD5L7mTRoxa+qkaPX9/QM4efoM65YtJjwigktXvHn+/Dmr1q2nbcsPkqRfKZGtvAlFgrOVMrjloESjeuyf9c+IKGuh/FzZsx+AC9t2UrZF0xjHPb3vw62TpwEIDgjg/oVLZHAzTx2v0bUzW8b+SJjlvXMvFkoKDw3FwTE1qZwcCQ8NJVPePGRwyxHZljCeM2fPkcvDHQ93N1I5ONCo3nts37UnWp2/rl6jUkXzuyLz5cnNnbv3ePDwIWB+xDt9+nTR6iuTIjQ0DK01wcHB2NvbM2veQtq3/hAHB8lIxsVWnhCU4GylVj+NZWXfIdHevn337AVKNW0IQNmWzXDxiH/qU8ZcOfEoU5Jrh48B5se581d7h36HdtBn10ZylS8LwOYxE2k3YzK1v+zGrikz8Bw9hHWDX/4yUJF8fHz9os1Xzpo1Cz5+0d+6XrhgAbZt3wmYg/nde/e5H8/LEpzTpOG9OrVo1voj3HPkIK2zM2fPn+fdWjUSpxM2wmRSVm9GJsHZCiUa1cff9wE3T5yKVj7/k27U6O7FgGO7SZ02LWEhobGfAPPrpbxWLGDZl/157u8PgMneHieXDIyrVJuV3wymy7JfAbh9+k/GV67Dj7Ubkylvbv6+ew+U4tOlc/l4wUzSZsmcWF0V/5Em5uytf//V9/q4A0/9/fH8sB0Lli6jSKGC2NvZxXveLp06sOa3RfT/6ksmTfuFnl0/4/eVq+nVdwDTZs5OwB7YDmWyfjMy+beRFfJVeZuSTRtQvGFd7FOnxjFdWj5eMJO57bswuV4zwPyG7BKNYl+kxmRvj9eKhRxZtIxTq9ZFlj+5fZdTK9cCcP3ocXSExjlTRgIePIys03BQX2Z92InWU75n3dDvyJg7F7V7fs6aQSMTr8PilWXLkoX7Pj6R3318fMmSOfovUWdnZ8YMHwKYV6Or06gZ7m7Wrd19/qL5SdLcuXIyevwPLJozg979vuX6jZuxLpr0JjP6jT5rGfx3hzGsHjicAR5F+DZPCWa3/piLO/Ywt32XyEe3lVI0HPQNe36JfSTTYfZU7l+4xPYfp0YrP7V6PYVqm/+JmqVAfuxSOUQLzJU7tuXshi08e/KEVE6O6AiNjogglZNTIvVU/FclihXl+s1b3Lpzh5DQUDZs2UrtmtWi1Xnq7x/5tu3fV62hfNnSkes4v8yLUXNYWBjhltSayaR4/vx5wnbEBtjKDUEZOb+GCm1aUqN7FwBOrlzLgbkLAUifPRvtZ01hSqMPyFelEpU6tOH2mbN8e3If8M+bUQ7MWUCHOdMY/OchwkNCmNfx88hzOzg6UqljWya91wyAPyZO4bMVCwgLCWF2m85J21HxUvb29gzp9w2fdutJeEQELTybUCBfPpb8vgKANi1b8NfVa/QbPByTnYn8efMweuigyOPjesQb4I+duyhRrChZLemsMiVL0KRlGwoWyE9hWYg/BlsZOcvj2yJZyOPbIlYJ8Pj2teIFrY45ec5eNmwkl5GzEMKm2Bl8Foa1JDgLIWyKraQ1JDgLIWyKjcRmCc5CCNsiwVkIIQzIVhbbl+AshLApckNQCCEMSNIaQghhQDJbQwghDMhGYrOsrSGEsC0J+SYUpVQGpdRypdRFpdQFpVRlpZSrUmqbUuqK5adLlPoDlFLeSqlLSql6UcrLKaX+tOybrKxoXIKzEMKmJPDCR5OAzVrrwkAp4ALQH9iutS4AbLd8RylVFGgNFAPqA9OUUi/WhJ0OeAEFLFv9lzUswVkIYVNMdsrqLT5KqXRAdWA2gNY6RGv9BPAEXrwSaR7QzPLZE1iqtQ7WWl8DvIGKSqnsQDqt9UFtXsxofpRj4u7HK/ZbCCEM7VXSGkopL6XUsSibV5RT5QX8gLlKqZNKqVlKqTRAVq31PQDLzyyW+m5A1BW9blvK3Cyf/10eL7khKISwLa8wz1lrPQOYEcdue6As0ENrfVgpNQlLCiMOsTWs4ymPl4ychRC2JeGSzreB21rrw5bvyzEHax9LqgLLT98o9T2iHO8O3LWUu8dSHi8JzkIIm5JQszW01veBW0qpQpaiOsB5YC3Q0VLWEVhj+bwWaK2UeksplQfzjb8jltSHv1KqkmWWRocox8RJ0hpCCNtil6Bjzh7AIqVUKuAq8DHmQe0ypVRn4CbQEkBrfU4ptQxzAA8Dumutwy3n6Qr8CjgCmyxbvCQ4CyFsSkIufKS1PgWUj2VXnTjqjwZGx1J+DCj+Km1LcBZC2BYbeURQgrMQwqbIkqFCCGFEMnIWQggDkpGzEEIYj0rY2RrJRoKzEMK2SFpDCCGMR9nGwFmCsxDCxsjIWQghjEem0gkhhBHJyFkIIYxHZmsIIYQRSVpDCCEMSNIaQghhPNa8VTslkOAshLAtktYQQgjjkRuCQghhRJLWEEII45GHUIQQwohk5CyEEAYkI2fr/BJwM7GbEClQT+ecyX0JwoAmR/z92ueQqXRCCGFEMltDCCEMSEbOQghhQBKchRDCgEyS1hBCCOORkbMQQhiQBGchhDAgO7vkvoIEIcFZCGFbZOQshBAGJMFZCCEMSIKzEEIYkEylE0IIA5LgLIQQBiRpDSGEMB4lI2chhDAgGTkLIYQB2Uhwto3xvxBCvKCU9ZtVp1N2SqmTSqn1lu+uSqltSqkrlp8uUeoOUEp5K6UuKaXqRSkvp5T607JvsrLijQASnIUQtsXOzvrNOr2AC1G+9we2a60LANst31FKFQVaA8WA+sA0pdSLRqYDXkABy1b/ZY1KcBZC2JYEHDkrpdyBRsCsKMWewDzL53lAsyjlS7XWwVrra4A3UFEplR1Ip7U+qLXWwPwox8RJgrMQwra8QnBWSnkppY5F2bz+dbafgL5ARJSyrFrrewCWn1ks5W7ArSj1blvK3Cyf/10eL7khKISwLa8wlU5rPQOYEds+pVRjwFdrfVwpVdOK08U2FNfxlMdLgrMQwrYk3GyNKkBTpVRDIDWQTim1EPBRSmXXWt+zpCx8LfVvAx5RjncH7lrK3WMpj5ekNYQQtiWBcs5a6wFaa3etdW7MN/p2aK0/AtYCHS3VOgJrLJ/XAq2VUm8ppfJgvvF3xJL68FdKVbLM0ugQ5Zg4ychZCGFbEn+x/bHAMqVUZ+Am0BJAa31OKbUMOA+EAd211uGWY7oCvwKOwCbLFi8JzkII25IID6ForXcBuyyfHwJ14qg3GhgdS/kxoPirtCnBWQhhW2zkCUEJzkII2yILHwkhhAHJyFkIIQzIJG/fFkII4zHJyFkIIYxHSc5ZCCGMR3LOQghhQDJbQwghDEhGzkIIYUAyW0MIIQxI0hpCCGFAktYQQggDkql0tqlI+coUzJ8v8vvUieNxz5Ej1rplqtTk5P5dr9Ve/6Ej2H/oCNvXrSRVqlQ8evyEDz7qxI4Nq1/rvCJxOLm68MUfawFIly0rEeHhBPg9AOCHt2sTHhr62m302LGedNmzEfb8OcEBgSzu3B3fy96vfd43hjyEYptSv/UWa5YuTNI27exMLF+zjrYtWyRpu+LVPXv0mPFlqwHQYGh/ggMC2fHDz5H7TXZ2RISHx3W41eZ/1IVbx0/yTpdOeI4fycxmbV77nG8MuSH4Zgh89oxuvb/hqb8/YWFh9Or2Ge/WrBGtjq/fA3r3/5aAwEDCw8MZNqAv5cuWYd/BQ/z8y0xCQkPxcHdjzLDBpHFyitFGx7atmbdoCa3e94yxb9a8BWzatp2QkFDq1qpBz67m909OnTmbdZu2kD1rVlwypKdYkcJ07vBR4vxHEPFqN2cazx4/xr10SW6dPE2wf0C0oN3/zEFmNPmQRzduUr5dK2r0+By7VA7cOHKcZd36oCMi4jy395791OzVFQDP8SMpUv9d0Joto7/n5LKVpMuWlU5L55I6XVpM9vYs69aHq/sOJkm/DUvSGrbpeXAwnq3NQc7dLQeTxn3H1B/G4ezszKPHT/iwY2fq1KiOinLTYf3mLVStXImun35MeHg4Qc+f8+jxE6bPmsvcX6bg5OjIjF/nM3fhYr7w+jRGm9mzZaVs6VKs2bCJWtWrRZbvO3iIGzdvsXzBXLTWdP3ya44eP0lqx9Rs3b6T1YvnExYeTvO2HShWpHDi/8cRccpcID9T6nqiIyJoMLR/rHWyFi5I2VbN+bHqe0SEhdFy6g+Ub9eKowuWxnne4k0acPfP85Rq3hS3UiUYV7oKzpky8tWRnfy1Zz/l2rbk4tYdbP3ue5TJRKpYfvm/cSStYZv+ndYIDQ1j4pTpHD1xCpNJ4ePnx4OHj8icKWNknRJFizJw+CjCwsJ4t1YNihQqyM7j+/C+do02H3exnCeU0iVLxNnu5590omvvr6lZrUpk2f5Dh9l/6AjN2rQH4NmzIK7fukVgYCB1alQnderUANECukgep5avjncEDFCwTg08ypXm6yM7AXBwdCTA1y/Wuh0WziQ0KIhH12+yvGdfavXpzomly9EREfj7+vHX7v3krFCWm0dP0Hb2VOwc7DmzegN3Tv+Z4H1LcWS2xpth3abNPHr8hJWL5uHgYE/tRs0IDgmOVqdCuTIsnP0Lu/fup+/gYXTu8BHp0qalytsVmThmlFXt5MrpQZFCBdm0dXtkmdbg9XEHWn/QPFrdXxcuef2OiQQVEvgs8nN4WBgqylxbB8svUaUUR+YvYd3A4S8934uc8wsqjoDz194DTKrRgGKN6tF+/v/Y/v3keEfibwQbSWvYRi8SkX9AABldXXBwsOfQ0WPcuXcvRp07d++R0cWFVs2b0cKzCecuXKR0yeKcOH2GGzdvARAU9JxrN27G29bnnT9mzoJFkd+rVn6bFWvXE/jM/Bffx9eXh48eUbZMKXbu3UdwcDCBz56xa9/+BOyxeF2Prt/EvUwpANzLlCJjnlwAXN6+m1ItPHHOnAkAJxcXXHJ6WHVO7z0HKNOqOcpkwjlTRvJVf4cbR47jktODAF8/Ds6ax6E5C/AoWypxOpWSmJT1m4HJyPklmjSoT9cvv6J5u44UKVSQvLlzx6hz5PgJZs9fiL29PU6OjowbOQxXFxfGDBtCn4GDCQkxT6/6svtn5MmVM862CuTLS9HChTh/8RIAVStX4q9r12ndyZyndnJ0ZMKo4ZQsVpTa1avRtPVHuGXPRvGiRUjr7JzwnRf/yekVa6nYvg19T+zl5rGTkdPg7l+4xIbBo+i2ZRXKZCIiNIzfv/iKx5Zf4PE5s2odeSpXpN+p/aA1a/oNxd/Hl4od2lD7655EhIYRHBDAgo6fJ3b3jM9GZmsorXXithD4JJEbeDMFPntGGicngoKe0+7Tzxg5aECKuinYM22u5L4EYUCTI/5+7eFs+JppVsccO89uhh0+y8g5hRoyagzeV68RHBLC+40bpqjALESiMni6wloSnFOoH74bmdyXIIQxyQ1BEdW9+z609+pKg+Yf0uiD1sxbbL5j/tO0X2jSqh2erT/ik2498PEzT53af+gwzdt2oEmrtjRv24GDR44l5+WLBFaj5+f0P3OQAX8einyI5IXaX/VgcsTfpMnoCkDOCmXpe2IvfU/spd/JfZRs1jiybplWzel3aj8D/jxE03EjkrQPKZZS1m8GJjnnBOLr9wC/Bw8oVqQwAYGBtGjXkakTx5MtSxacLTfr5i/5De+r1xjxbX/OX7xExoyuZM2cmcvef9G5ey/2blmfzL1IOracc85erAgdl8wxr7UREkLXTStZ1q03ft5XyeDuRpuZP5O1cAEmlK9B4MNHODg6Eh4SQkR4OOmyZaXfqf0MditE6vTp6HtiL9+Xr0HAg4e0mzudowuWcnnH7uTuYqJJkJzzplnW55wbfGrYCC0j5wSSJXOmyLyvc5o05M2TGx9fv8jADBAUFBQ5X7Vo4UJkzZwZMM/SCAkJJiQkJOkvXCS4rEUKcePwMUKDgogID8d7zz5Kvt8EgOYTx7Cm3xCiDope1AOwT506cl+mvHnwu/wXAQ8eAnB5+y5KtWiaxL1JgUx21m8GJjnnRHD77l0uXLpMqeLFAPhxynRWb9hIWmdn5s+YFqP+lu07KFKoEKlSpUrqSxWJ4N7Z8zQeNRgnVxdCg55TtMF73Dx+kuJNGvDk7l3unjkb45hcFcvRdvZUXHN5sKDDZ0SEh+PnfZWshQvgmisnT27foYRnY+xTOSRDj1IYg6crrPWfR85KqY/j2eellDqmlDo2Y86v/7WJFCnw2TN6ft2fgV/1jhw19/6iK7s3raNJg3osXPp7tPpX/rrK95OnMuLb2NdjECmPz8XL/DH+J7pvXUPXTSu4c+YsEWFhvDfwazYO+S7WY24cOc6YEpX4vmIt6vbvg/1bbxH05AnLuvWh09K59NqzmUc3bhAeFpbEvUmBTCbrNwN7nauL8xlUrfUMrXV5rXV5r086vUYTKUtoaBg9v+5Pk4b1ea9OrRj7G9evx9YdOyO/3/fx4Yuv+jJuxFByergn5aWKRHZozgImlK/O5JoNefboMY+u3yRjnlz0O7WPoVfPkMHdjW+O7yFt1izRjvO5eJmQwECyFy8KwNn1m5lYuQ4/VqmL7yVv/LyvJkd3UhYbuSEYb1pDKXUmrl1A1oS/nJRLa823I0aRN09uPv6obWT59Zs3yZ3T/FTgjj17yZvbfCPsqb8/Xj370KdHN8qVlkdubY1z5kwE+D3AxcOdUu83YeI7ddk9+ZfI/UOvnuH7CjUJfPgI19y5eHLrNhHh4bjk9CBLoQI8un4j2nkcM2SgatfOzP2wUzL1KAWxkal0L8s5ZwXqAY//Va6AA4lyRSnU8VOnWbNhEwXz549ccrTPF11Zvnot127cRCkTbtmzMfzbfgAs/O13bt66zbSZc5g2cw4Ac6ZNJqOra7L1QSSczssXkCajK+Ghofz+xdcEPXkSZ918VSvxbr/ehIeGoiM0y7p/ReDDRwC0+GkcbqWKA7B55Hj8rvyVFJefshn8Rp+14p1Kp5SaDczVWu+LZd9irXXbWA6L7g2ZSidejS1PpRP/XYJMpdvzm/VT6ap/aNjcRrwjZ61153j2vTwwCyFEUntD0hpCCJGyGPxGn7UkOAshbIuNjJxtoxdJaMCwkVSuU5/GLWN/G/LajZtp0qodTVq1o3WnT7l4+XLkvj37D1Lv/ZbUbdqCGXPnRZZPmDSFJq3a0XfwsMiy1es3Rq7PIYxLmUz0Pb4Xr7W/AeBWqgR9DvxB3xN7+frILnJWKBvrcY7p0/PJsvl8e/4oA88dIXelCgA0HPEt/U7tp++JvXTbvIp02bMBkOedt+l3aj9fHd5Jpnx5I8/RddPKJOhlyqKUsnozMgnOr6h5k8bMmvJTnPvd3XKwcNZ01i1bRNcunzB41FgAwsPDGTFuArN+/okNK5ayfvNWvK9exd8/gJOnz7Bu2SLCIyK4dMWb58+fs2rdBtq2/CCJeiX+q5q9unL/wqXI757jRrBpxFjGl63GxqGj8YxjsaLmP43lwpY/GF20AuNKV8HngvmX+I4JkxlXugrjy1bj7IbN1B9int1Tu08P5nzQnvXfjqBq108AqDf4G7aN+SGRe5gCmeyt3+KhlPJQSu1USl1QSp1TSvWylLsqpbYppa5YfrpEOWaAUspbKXVJKVUvSnk5pdSfln2TlRW/GSQ4v6IK5cqQPn26OPeXLVWS9OnM+0uXKM59H18Azpw9Ty53dzzc3Ujl4ECjenXZvmsPyqQIDQtDa03w82Ds7e2ZNX8h7du0wsFBsk5GlsEtB0Ub1uPg7PmRZVprUlv+/6dOn46/796PcVzqtGnJX71K5HHhoaEE/f03AM/9/SPrvZUmjflFkpY6Do6OpHJyJDw0jEx585A+Rw6898grymJIuNdUhQFfaa2LAJWA7kqpokB/YLvWugCw3fIdy77WQDGgPjBNKfViXt90wAsoYNnqv6xx+dufiJavXkv1KpUB8PHzJVu2f57byZolC2fOnsM5TRreq12LZm3aU7liBdI6O3P23AW+8Po0uS5bWKn5j2NZ228Ib6X9Z3Grlb3703XzSppNGIkymfixynsxjsuYNzcBfg9oN2cabqVKcOvEKVb06keI5V2RjUYNpmL71gT9/ZQptc3Lh24bO5HW/5tEaFAQCzp8hueEUWwcYt3Lg984CZRz1lrfA+5ZPvsrpS4AboAnUNNSbR6wC+hnKV+qtQ4GrimlvIGKSqnrQDqt9UEApdR8oBmwKb72ZeScSA4dPcby1ev4uucXQOQAKJoX/7Lp0qk9a5YupH+fXkya/j96dvXi91Vr6NVvINNmzUnKyxZWKtaoHv5+ftw6cSpaedWunVnVZyBDcxVjVZ+BtJ01JcaxJnt73MuWYt8vsxlfrhrBgYG827935P4Ng0YyNFcxji/+nWpfeAFw5/SfTHznXX6u04SMeXPz9N49UIpOS+bSfv4M0mbJnKj9TVES4fFtpVRuoAxwGMhqCdwvAviLZ/DdgKgvhLxtKXOzfP53ebwkOCeCi5evMGjkd0z7cQIuGdIDkC1LFu7f94ms4+PrSxbLW5hfePFi19y5crJ6/UYmjfuOK95/cf1m/G/tFkkvb5VKlGjSgKFXz9BpyRwK1q5O+/kzqNihDadXrgXg5O+ryFUx5g3BJ7fv8OT2HW4cOQ7AqeVr8CgT8xH+Y4t/p1TzmEuE1vv2GzaPnECDIf3ZOOw7ji36jeo95cWukZTJ6i3qIm2WzSvG6ZRyBlYAX2qtn8bXcixlOp7yeElwTmB3792nx9f9GT9yWLQ3bZcoVoTrt25x685dQkJD2bBlG7VrVI927KRp5lFzWFgY4RERAJhMJp4/f56kfRAvt27gcIbkLMrwvCX5tc0nXN6xhwUdvPj77n3y16gKQMHaNfC7EnOhIn8fX57cukOWgvkBKFSnRuRNxcz580bWK9G0Ab4Xr0Q7tmLHtpzbuIWgJ09wcHJER0SgIzSpnBwTq6spzyuMnKMu0mbZZkQ/lXLAHJgXaa1fTI3xUUplt+zPDvhaym8DHlEOdwfuWsrdYymPl+ScX1GfAYM4cvwEj588oXr9xvT43BxMAdp80JypM2fz5O+/GT5mPAB2dnasXDQPe3t7hvT7mk+79yQ8IoIWTZtQIN8/fxH/2LmbEsWKRi7AX6ZkcZq0akvBAvkpXLBg0ndU/CdLvXrS4qdxmOztCH0ezNLPegGQLns22sz8mf81bgnA8p596bBwFnapHHh49TqLPukOQJMxw8lSKD86IoLHN27xW9d/0h0Ojo683aENU+u9D8DOH6fSefkCwkNC+bXtJ0ncUwOzS5i1NSwzKmYDF7TWE6PsWgt0BMZafq6JUr5YKTURyIH5xt8RrXW4UspfKVUJc1qkA/DzS9uX11SJ5CBra4jYJMTaGhFndlodc0wla8XZnlKqKrAX+BOIsBQPxBxglwE5gZtAS631I8sx3wKfYJ7p8aXWepOlvDzwK+CI+UZgD/2S4CsjZyGEbUmgh0ssC77FdbI6cRwzGhgdS/kxoPirtC/BWQhhW2zk8W0JzkII22Lwx7KtJcFZCGFb7GwjrNlGL4QQwsLoCxpZS4KzEMK2SM5ZCCEMSEbOQghhQDJyFkIIA5KRsxBCGFACPb6d3CQ4CyFsi6Q1hBDCgCStIYQQRiTBWQghjEdGzkIIYUASnIUQwoDkhqAQQhiQbQycJTgLIWyNbURnCc5CCNsiOWchhDAgCc5CCGFAckNQCCGMSEbOQghhPJLWEEIIA5LgLIQQRiTBWQghDEde8CqEEEYkszWEEMKAZOQshBAGJMFZCCGMSIKzEEIYj4ychRDCgGwjNktwFkLYGJmtIYQQBiRpDSGEMCIJzkIIYTwychZCCAOS4CyEEAZkIzcEldY6ua/hjaGU8tJaz0ju6xDGIn8uRGxs41dMyuGV3BcgDEn+XIgYJDgLIYQBSXAWQggDkuCctCSvKGIjfy5EDHJDUAghDEhGzkIIYUASnIUQwoAkOCcRpVR9pdQlpZS3Uqp/cl+PSH5KqTlKKV+l1NnkvhZhPBKck4BSyg6YCjQAigJtlFJFk/eqhAH8CtRP7osQxiTBOWlUBLy11le11iHAUsAzma9JJDOt9R7gUXJfhzAmCc5Jww24FeX7bUuZEELESoJz0ohtmSyZwyiEiJME56RxG/CI8t0duJtM1yKESAEkOCeNo0ABpVQepVQqoDWwNpmvSQhhYBKck4DWOgz4AtgCXACWaa3PJe9VieSmlFoCHAQKKaVuK6U6J/c1CeOQx7eFEMKAZOQshBAGJMFZCCEMSIKzEEIYkARnIYQwIAnOQghhQBKchRDCgCQ4CyGEAf0f0+pLOmuHVLsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lsvc_model = lsvc_search.best_estimator_\n",
    "lsvc_model.fit(X,Y)\n",
    "\n",
    "lsvc_predicted = lsvc_model.predict(X_valid)\n",
    "lsvc_matrix = confusion_matrix(y_valid, nb_predicted) \n",
    "# print(lr_matrix)\n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in lsvc_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in lsvc_matrix.flatten()/np.sum(lsvc_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(lsvc_matrix, annot=labels, fmt='', cmap='Reds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "4afacce0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.98      0.98      0.98      9738\n",
      "         1.0       0.98      0.98      0.98      9671\n",
      "\n",
      "    accuracy                           0.98     19409\n",
      "   macro avg       0.98      0.98      0.98     19409\n",
      "weighted avg       0.98      0.98      0.98     19409\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "lsvc_report = classification_report(y_valid, nb_predicted) \n",
    "print(lsvc_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08a7ac8f",
   "metadata": {},
   "source": [
    "#### All models perform well on the validation set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13037e79",
   "metadata": {},
   "source": [
    "## Apply Models to the test data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "896c2be0",
   "metadata": {},
   "outputs": [],
   "source": [
    "testfile = 'test_imperson_without4n7_balanced_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "abc42c1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 40158 entries, 0 to 40157\n",
      "Columns: 152 entries, 1 to 155\n",
      "dtypes: float64(41), int64(111)\n",
      "memory usage: 46.6 MB\n"
     ]
    }
   ],
   "source": [
    "#remove column 38\n",
    "test = pd.read_csv(testfile)\n",
    "test_data = test.drop(labels = '38', axis =1, inplace = False)\n",
    "test_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9c92ecc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40158, 151)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = test_data.values\n",
    "x_test = array[ : , 0:151]\n",
    "y_test = array[ : , 151]\n",
    "x_test.shape\n",
    "#y_test.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a20d1a7e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40158, 15)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1) Normalise data \n",
    "n_X_test = norm_scaler.transform(x_test)\n",
    "\n",
    "#2) predict with encoder\n",
    "X_test = encoded.predict(n_X_test)\n",
    "\n",
    "#3) scale encoder output\n",
    "X_test = norm_scaler20.transform(X_test)\n",
    "\n",
    "#4) add 20 features from encoder output to normalised full dataset (use diff names so doesn't over-write and can add)\n",
    "X_test_full = pd.DataFrame(n_X_test)\n",
    "X_test_gen = pd.DataFrame(X_test)\n",
    "X_test_df = pd.concat([X_test_full,X_test_gen],axis = 1)\n",
    "\n",
    "#5) feature selection 1 - variancethreshold\n",
    "X_test = feat_sel.transform(X_test_df)\n",
    "\n",
    "#6) feature selection 2 - Kbest with chi2\n",
    "X_test = kbest.transform(X_test)\n",
    "\n",
    "#7) feature selection 2 - Kbest with chi2\n",
    "X_test = kbest_mi.transform(X_test)\n",
    "\n",
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50b58d9e",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "84711c88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7295931072264555"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_pred = lr_model.predict(X_test)\n",
    "lr_score = lr_model.score(X_test, y_test)\n",
    "lr_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "b56756cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.971\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAkBklEQVR4nO3deXxU5b3H8c8vO4QQIGFPwhJ2RBQCiKgoiiJVKa2t29VWrWirtpa2Smuttvb29l5bXOpCqVpFq2gVlVqq4gYqIouyIxD2CLITlpBt5rl/TNAkhGRCJjOZM9/36xXNc+bkzO+QvL558syZ3zHnHCIiEv3iIl2AiIiEhgJdRMQjFOgiIh6hQBcR8QgFuoiIRyRE6okzMzNd165dI/X0IiJRafHixbudc21reixigd61a1cWLVoUqacXEYlKZrb5eI9pyUVExCMU6CIiHqFAFxHxCAW6iIhHKNBFRDyizkA3syfNbKeZrTjO42ZmD5lZvpktM7NBoS9TRETqEsxli08BDwPTjvP4hUDPio9hwGMV/48uWxfAs5dCSSHEJ0O/cbBhDpQegj7fCOyz+nVISYOzfwV53w9sW/QUzH8U9m0Cfxl0PBUycr/et2UW7NsAPUZDlxGw+jXoOy7w9YueCoybZ0LR7q+3i4h3bV0Amz6ArmdC9tCQHtqCaZ9rZl2B151zJ9Xw2F+B951zz1eM1wBnO+e213bMvLw8F/br0O9Jr2GjQUIzKC+q37EGXgnOD8umn1gtHU+F7Z8du73/tyGnht+H9W5zXI/9G/PY9T5+tB67nseP6n/zRts5ev/Ngzy2r7CAuGUvYs4PCcnwvZn1DnUzW+ycy6vpsVC8sagzsLXSuKBi2zGBbmYTgAkAOTk5IXjqeqgxzAFc/cMcYOlzDSqnxjAHWPly4ENEoozV+qgDrPIvCV9pYKYewll6KAK9prOo8deVc24qMBUCM/QQPHdwjhvmDTDqN4H/v/u7E/v6vuMCyy3Vnf8HOOWKE6+rMqv9B6zazo147HoeP1qPXe/j69+8wceu7/HrXUvDFZf5ePCddUydu4Gzmm3kcX5HvL8M4pMCyy4hFIpALwCyK42zgG0hOG7kWDyc9O3g1tCbZ2gNXUSOa8Izi5m7dhffGZzFr78xmvg9Q5r0Gvo3gFuAsQReDH3IOVdnlWFbQ582Hja8W8sOcdAsHVLbBQK57yUw+reNX5eIeNahknIS4oyUxHg+Xr+Hcr+fM3vW2E+r3hq0hm5mzwNnA5lmVgDcDSQCOOemALMIhHk+UARcG5KqQ+V4YX5PYXjrEJGYMGftLn41YznfPLUTv7igD8NzM8L23HUGunOu1gVdF5ji3xyyikREotD+olLufX01L39aQG7bVEb1aRf2GiLWPjeiNDsXkRD6KH83P5m+hP1FpdxyTg9uGdWDlMT4sNcRm4EuIhJCGS2SyG7TjKevG0L/To1wVV2QFOgiIvXknOOlxQWs3HaAey7pT58OLZnxw9OxCFwWWZm3A33rgkhXICIes3VvEb96ZTkfrNvN0K5tKC7zkZIYH/EwB68H+hPnR7oCEfEIn98x7eNN/N8ba4gzuPebJ3HV0Bzi4iIf5Ed5O9BresNqfHL4yxCRqLf3cCmTZ69lWPc2/Pf4AXRu1SzSJR3D44Feg7t2RroCEYkSZT4/r372Bd8elEXbtGT+feuZZLdp1iSWV2oSe4EuIhKE5QWF/OKlpXz+5UHatUxhZK+25GQ0j3RZtVKgi4hUUlzm44G31/G3DzaQkZrEX68ezMheoXnbfmNToIuIVHLDtEV8sG43lw/J5pdj+5LeLDHSJQVNgS4iMe9gcRmJ8XGkJMZz8zk9uGlkLiN6ZEa6rHrTTaJFJKa99/lOLrh/Lg+9sw6A07pnRGWYg2boIhKj9h4u5d7XV/HKZ1/Qs10LzuvXPtIlNZgCXURizgfrdnHb9CUUHinjx+f25OZzcklOCH8zrVBToItIzGmXlkK3zFR+P/4k+nRoGelyQsa7a+jTxke6AhFpIpxzTF+whbteXQFA7w5p/POm4Z4Kc/DyDL3W286JSKzYsqeISTOWMW/9Hk7r3rSaaYWadwO9JurjIhIzfH7H3z/ayJ/eWkNCXBx/GD+Ay4dkN6lmWqEWW4GuPi4iMWPv4VIefGcdI3Iz+f34k+iY3vSaaYVabAW6iHhaaXmgmdalgwPNtGb9+EyyWjfdZlqhpkAXEU9YunU/t7+0jDU7DtIhPYWzerUlu03TbqYVagp0EYlqR0p9TJ69hic+3Ei7tBQevyaPs6KkmVaoKdBFJKrdMG0RH+bv5oqhOfxybB9apkRPM61QU6CLSNQ5UFxGUkUzrVtH9eBH5+Ryem509l8JJe++sUhEPOmd1Ts4f/JcHqxopjWse4bCvIJm6CISFfYcKuG3/1rFzKXb6NMhjTH9O0S6pCbHw4FuVL1JdGxctiTiRXPX7uK2F5ZwsLiMn57Xix+enUtSghYYqvNuoKdnQ+GWqmMRiUod0lPo0bYFvx9/Er3ap0W6nCbLu7/iDm6rfSwiTZbf73juky3c+cpyAHq1T+PFm4YrzOvg3Rn6MUssWnIRiQabdh9m0oxlzN+wl+HdM75qpiV1826g+8tqH4tIk+LzO578cCN/nr2GxLg4/vitAVw2JDtm3rYfCkEtuZjZGDNbY2b5ZjaphsfTzexfZrbUzFaa2bWhL7W+NEMXiSZ7D5fyl3fXcUaPtsyeOJLLh+YozOupzhm6mcUDjwCjgQJgoZnNdM6tqrTbzcAq59zFZtYWWGNm/3DOlTZK1cGISwR/adWxiDQpJeU+Znz6BZflZQeaaf3kTDq3ip1mWqEWzJLLUCDfObcBwMymA+OAyoHugDQLfBdaAHuB8hDXWj/+0trHIhJRn23Zxx0vL2PtjkN0btWMs3q1Jat1bDXTCrVgAr0zsLXSuAAYVm2fh4GZwDYgDbjMOeevfiAzmwBMAMjJyTmRekUkyhWVlvPnt9by5Ecb6dAyhb9/f0jMNtMKtWACvaa/fVy18QXAEmAUkAvMNrMPnHMHqnyRc1OBqQB5eXnVjyEiMWDCtMV8mL+b/zothzvG9CEthptphVowgV4AVH5XThaBmXhl1wJ/dM45IN/MNgJ9gAUhqfJEWDw4X9WxiERE4ZEykhMCzbR+fG5Pbh3Vg2HdMyJdlucEc5XLQqCnmXUzsyTgcgLLK5VtAc4FMLP2QG9gQygLrbfKYQ5w7AqQiITB7FU7OP/+OTzwdqCZ1tBubRTmjaTOGbpzrtzMbgHeBOKBJ51zK83sporHpwD3Ak+Z2XICSzR3OOd2N2LdJ0ArPCLhtPtQCffMXMnry7bTp0MaYweomVZjC+qNRc65WcCsatumVPp8G3B+aEsTkWj1/pqd3PbCEopKfPxsdC9uOjuXxHjvdhppKrz7TtHE5lBW9PU4ST0gRMKlU6tm9G6fxu+/eRI91X8lbLz7K7PbWVXHvS+MTB0iMcDvdzwzfzO/nPF1M60XbhyuMA8z787QCxZWHW+cE5k6RDxuw65DTHp5OQs27eXMnplqphVB3g304sKq46J9kalDxKPKfX7+9sFG7n97LSkJcdx36clcOjhLb9uPIO8Gur9a54HqlzGKSIPsKypjypz1nNO7LfeOO4l2LVMiXVLM82agTxt/7DZdhy7SYCXlPl5aXMAVQ3Jom5bMf35yJp1aNYt0WVLBm4G+4d1jt6V1DH8dIh6yeHOgmVb+zkN0aZPKGT0zFeZNjDcDvSbffTrSFYhEpcMl5fzprTU8NW8TndKb8fR1QzmjZ2aky5IaxE6gZw+NdAUiUWnCM4v4KH8P3xvehV+M6UOL5NiJjWjj0e+MUfWt/nrVXaQ+CovKSE4MNNO67bxe3HYeDOnaJtJlSR08+sai6n1b1MdFJFhvrNjOeffP4f631wKBIFeYRwePztBFpL52Hizm7tdW8p8VX9KvY0suPrlTpEuSelKgiwjvrdnJbdOXcKTMxy8u6M2Es7qrmVYU8migaw1dpD6yWjWjf6eW/G7cSfRo1yLS5cgJ8uav4LiE2sciMc7vdzw9bxOTXl4GQM/2aTx3w2kK8yjnzUDvP772sUgMW7/rEN/968fcPXMl2wqLKS5TWwyv8ObUtcsIWP5i1bFIjCvz+Zk6dwMPvrOOZonx/Ok7A/n2oM5qpuUh3pyhf/JY7WORGFR4pIypczdwXt92zJ54ljojepA3Z+h7N9Y+FokRxWU+/rloK1cN60Jmi2TeuO1MOqar/4pXeTPQfSW1j0ViwMJNe7njpWVs2H2YbpktOKNnpsLc47wZ6CIx7FBJOf/3xudM+3gzWa2b8cz1aqYVK7wZ6HGJ4C+rOhaJEROmLeLjDXu4dkRXfn5+b1LVTCtmePM7ndYRCrdUHYt42P6iUpIT4mmWFM/Pzu8FGIO7tI50WRJm3rzKpWhP7WMRD5m1fDvnTZ7DAxXNtAZ3aaMwj1HenKGXHa59LOIBOw8Uc9drK3hz5Q4GdE5n3CmdI12SRJg3A13E4979fAe3TV9CSbmfSRf24QdndCNBzbRinjcDPaE5lBdVHYt4SE6b5gzMbsVvL+lP97bqvyIB3vyVnppZ+1gkyvj8jic/3MjtLy0FoEe7NJ65fpjCXKrw5gy98hUuNY1Fosi6HQe54+VlfLplP+f0bktxmY+UxPhIlyVNkDcDXcQDSsv9/HXOev7ybj6pyfE8cNkpjDulk/qvyHEFteRiZmPMbI2Z5ZvZpOPsc7aZLTGzlWY2J7RlisSeA8VlPPHRRs7v357ZE0fyzVPVGVFqV+cM3czigUeA0UABsNDMZjrnVlXapxXwKDDGObfFzNo1Ur3BScuCgwVfj9NzIleLSD0Ul/l4YeFWrj4t0EzrzdvOon3LlEiXJVEimCWXoUC+c24DgJlNB8YBqyrtcyUwwzm3BcA5tzPUhdZL1mBYXSnQc06LXC0iQfpkwx4mzVjOxt2H6dGuBSN6ZCrMpV6CWXLpDGytNC6o2FZZL6C1mb1vZovN7JqaDmRmE8xskZkt2rVr14lVHIyN1VZ88mc33nOJNNDB4jJ+/epyLps6n3K/n3/8YBgjeujKLKm/YGboNS3auWrjBGAwcC7QDPjYzOY759ZW+SLnpgJTAfLy8qofI3RaZcOX+78et+7eaE8l0lATpi1m/sY9XH9GN352fi+aJ+laBTkxwfzkFADZlcZZwLYa9tntnDsMHDazucBAYC2RUKK3/kvTtvdwKc0SA820fn5Bb8xgUI76r0jDBLPkshDoaWbdzCwJuByYWW2f14AzzSzBzJoDw4DVoS21Hg7tqDou/CIydYhU45xj5tJtnDd5Dvd/1UyrtcJcQqLOGbpzrtzMbgHeBOKBJ51zK83sporHpzjnVpvZG8AywA887pxb0ZiF16r6HYrKdcciibwvC4v59asreHv1DgZmpfOtQWqmJaEV1GKdc24WMKvatinVxvcB94WutIaotuyva3clwt5ZHWimVeb3c+fYvlx3Rjfi4/RzKaHl0Vdfqr3e6hrv9VeRYHTJSGVQl9b89pL+dM1MjXQ54lHebM5VPcA1Q5cw8/kdj3+wgZ+9eLSZVguevm6owlwalfdm6FsXgPNFugqJYWt3HOT2l5axZOt+RvVpp2ZaEjbeC/TXJx67rW2v8NchMae03M9j76/n4ffWkZaSyIOXn8IlA9VMS8LHe4G+q4arJb8xOfx1SMw5UFzGU/M2MnZAR35zUT8yWiRHuiSJMd4LdH8Nyy3ZQ8Nfh8SEI6U+nl+whe+d3vWrZlrt1H9FIsR7gX4M/bkrjWPe+t1Menk5W/YW0btDGiN6ZCrMJaK8F+hxCeAvqzoWCaEDxWX8z6zPeX7BFrpkNOf5G05jeG5GpMsS8WCgdzgZti2uOhYJoQnTFrFg415uPKs7t53Xi2ZJuoJFmgbvBXr1RlxqzCUhsOdQCc2TEmiWFM/tY/oQb8bA7FaRLkukCu+9sejI/trHIvXgnOO1JV9UaaY1KKe1wlyaJO/N0EVCZHvhEX79ygre+Xwnp2S34tLBWZEuSaRW3gv00sO1j0WCMHvVDn76whJ8fsddF/Xj+6d3VTMtafK8F+hlRbWPRYLQLTOVvK6t+d0lJ5GT0TzS5YgExXtr6NUbK6rRogSh3Odn6tz1THxhCRBopvXUtUMV5hJVvBfoqZm1j0WqWb39AN96bB5/mPU5B0vKKS5TczeJTt5bcuk+Epa/WHUsUoOSch+PvLeeR9/Lp1XzRB65chBjB3RQMy2JWt4L9C+X1T4WqXCouJxn52/mkoGduOuifrROTYp0SSIN4r1A13XoUoui0nKe+2QL147oRkZFM622aeqKKN7gvUDXZYtyHB/l72bSjGVs3XuEfh1bcnqPTIW5eIr3Ar16+9ya2ulKTCk8UsYf/r2aFxZtpVtmKi9MOI1h3dVMS7zHe4GekAzlRVXHEtNufGYRCzft46aRudx2Xk/dDk48y3uBXnqw9rHEhF0HS0hNjqd5UgJ3jOlDQlwcA7LSI12WSKPy3nXocUm1j8XTnHPM+LSA0ffP4f7ZgWZap+a0VphLTPDeDL28qPaxeNYX+49w5yvLeX/NLgbltOKyIdmRLkkkrLwX6BKT3lr5JT99YQkOuOfiflw9XM20JPZ4MNDjAH+1sXiVcw4zI7ddC07rnsE9l/Qnu436r0hs8l7apaTXPhZPKPf5eez99fy0oplWbtsWPPH9IQpziWnem6HrKhfPW7XtALe/vJQVXxzggv7tKS7z6VJEEbwY6ElpULyv6lg8objMx8Pv5jNlznpaNU/isasGceGAjpEuS6TJ8F6gV38dTK+LecbhknKeW7CFcad05q6L+tKquS5JFaksqDV0MxtjZmvMLN/MJtWy3xAz85nZpaErsZ5ad699LFHlcEk5U+eux+d3ZLRIZvZPz+LP3x2oMBepQZ0zdDOLBx4BRgMFwEIzm+mcW1XDfv8LvNkYhQbtQEHtY4kac9fu4pczlrOt8AgndU7n9NxMMlqolYPI8QQzQx8K5DvnNjjnSoHpwLga9rsVeBnYGcL66u/w7trH0uTtLyrl5/9cyjVPLiA5MY5/3jic03N15ymRugSzht4Z2FppXAAMq7yDmXUGxgOjgCHHO5CZTQAmAOTk5NS31uA4X+1jafImPLOYxZv3cfM5udw6Ss20RIIVTKDX9LJi9VsvPwDc4Zzz1Xb7LufcVGAqQF5enm7fLF/ZebCYFskJNE9K4Fdj+5IYb/TvpPcQiNRHMIFeAFRuipEFbKu2Tx4wvSLMM4GxZlbunHs1FEWKdznneGlxAb//92q+MziLX1/Uj1OyW0W6LJGoFEygLwR6mlk34AvgcuDKyjs457od/dzMngJej1iYW3zVZRbTn+tN1da9RfzqleV8sG43Q7q25ophjbQMJxIj6gx051y5md1C4OqVeOBJ59xKM7up4vEpjVxj/VicAj0KvLHiSya+uAQDfjeuP/81rAtxaqYl0iBBvbHIOTcLmFVtW41B7pz7fsPLaoDqa/i1rOlL+B1tptWrfQtG9Mjk7ov7kdVa/VdEQsF7zbmcv9pYr702BWU+P4+8l89Ppi8BoHvbFvztmjyFuUgIeS/Qq5+SZugRt+KLQsY9/BH3vbkGn3OUlOtSUpHG4L1eLv7y2scSNsVlPh58Zx1T526gTWoSf716MBf07xDpskQ8y3uBblb1Knnz4B8hUaKo1MeLC7fy7UGduXNsP9KbJ0a6JBFP816gt8yCws1fjzucHLlaYtChknKenb+ZG87sTpvUJGZPHEmbVDXSEgkHbwX61gVVwxxg1+rI1BKD3l+zkztfWcG2wiMMzGrF8NwMhblIGHkr0F+feOy2sqLw1xFj9h0u5d5/r2LGp1/Qo10LXrrpdAZ3aR3pskRijrcCfd+mY7dl9g57GbHmxmcX8+nmffx4VA9uHtWD5AS9mUskErwV6L0vhOUvfj1OSoNbFkSuHg/beaCY1OQEUpMTuHNsXxLj4+jXqWWkyxKJad66BGTP+qrjzF6RqcPDnHO8uHAr506ew+TZawEYmN1KYS7SBHhrhr7t09rH0iBb9gSaaX2Yv5uh3dpwlZppiTQp3gp0jGoXoUeqEM95Y8V2fvrCUuLjjN9/8ySuHJqjZloiTYy3Aj0lHYr3VR1LgxxtptW7Q0tG9mrLby7uR6dWzSJdlojUwFtr6CWFtY8laKXlfv7yzjp+PH0Jzjm6ZaYy5erBCnORJsxbgX5Mp0V/zftJrZYV7OeShz/kzxUvepb69O8oEg28teQiDVJc5uP+2Wv52wcbaJuWzN+uyWN0v/aRLktEguStQNft5xqkqNTHS4sLuGxINpMu7Et6MzXTEokm3gr0uATw+aqOpVYHi8t4Zv5mbjwrlzapSbw9cSSt1X9FJCp5K/F8JbWPpYp3P9/Bna+sYMeBYk7Nbs3w3AyFuUgU81agxydXDfH45MjV0oTtOVTC715fxWtLttGrfQsevep0Ts1RMy2RaOetQG/WGg59WXUsx/jhs5/y2dZ93HZeT350dg+SErx1sZNIrPJWoMtxfVlYTFpKoJnWXRf1Iykhjt4d0iJdloiEkLemZod31T6OQc45nl+whdGVmmkNyEpXmIt4kLdm6M5X+zjGbN5zmEkvL+fjDXsY3j2Da4Z3iXRJItKIvBXo8pVZy7cz8cUlJMbF8T/fGsDlQ7IxUzMtES9ToHvM0WZafTu2ZFSfdtx1UT86pqv/ikgs8NYa+jHtcmNnRlpa7ueBt9dyy/OffdVM69GrBivMRWKIxwLd1TH2piVb93PxXz7kgbfXkRBnaqYlEqO05BLFjpT6mDx7DU98uJF2aSk88b08zu2rZloisUqBHsWKy3y88tk2rhiaw6QL+5CWomZaIrEsqCUXMxtjZmvMLN/MJtXw+FVmtqziY56ZDQx9qQJwoLiMh99dR7nPT+vUJN6ZOJL/Hj9AYS4idc/QzSweeAQYDRQAC81spnNuVaXdNgIjnXP7zOxCYCowrDEKrqNavHxP0bdX7eDOV5ez62AJg7u0YXhuBunNFeQiEhDMkstQIN85twHAzKYD44CvAt05N6/S/vOBrFAWGbS4ePCXVxp7I+z2HCrhnn+t4l9Lt9GnQxp/uyaPk7NaRbosEWliggn0zsDWSuMCap99Xw/8p6YHzGwCMAEgJycnyBLro9qM3CNvpDnaTGvi6F7cNDJXzbREpEbBBHpNqVjj9YBmdg6BQD+jpsedc1MJLMeQl5cX+msK/WXVxuU17xcFthceoWVKIqnJCfzm4kAzrV7t1X9FRI4vmKleAZBdaZwFbKu+k5mdDDwOjHPO7QlNefUw++5jt0XhTaL9fsc/PtnM6Mlz+fNbgWZaJ3VOV5iLSJ2CmaEvBHqaWTfgC+By4MrKO5hZDjADuNo5tzbkVQZj3kPHbktsHv46GmDj7sNMenkZn2zcy4geGXz/9K6RLklEokidge6cKzezW4A3gXjgSefcSjO7qeLxKcBvgAzg0YoGUOXOubzGK7umQmuYjQ+9IawlNMS/lwWaaSUlxPF/3z6Z7+RlqZmWiNRLUG8scs7NAmZV2zal0uc/AH4Q2tJCYPRvI11BnY420+rfqSWj+7Xnrov60b5lSqTLEpEopMslIqSk3Mfkt9Zw83Of4pyja2YqD185SGEuIidMgR4Bn27Zx0UPfchD7+aTkhCvZloiEhLq5RJGRaXl/OnNtfx93kY6tkzh79cO4Zze7SJdloh4hAI9jErK/Pxr2TauPq0Lt4/pQ4tk/fOLSOgoURpZ4ZEynp63iR+dnUvr1CTenjiS9GbeaEkgIk2LAr0RvbnyS+56dQV7DpcyrFsbhnXPUJiLSKNRoDeCXQdLuGfmSv69fDt9O7bkie8NYUBWeqTLEhGPU6A3gh/9YzFLtxby8/N7cePIXBLjdTGRiDQ+BXqIfLH/COnNEmmRnMDdF/cnOSGOnuq/IiJhpKljA/n9jmkfb+L8yXOYXKmZlsJcRMJNM/QGWL/rEJNeXsbCTfs4s2cm147oGumSRCSGKdBP0OvLtjHxxaWkJMRx36Unc+lgNdMSkcjyRqDX1Au9kRxtpjWgczpj+nfg1xf1pV2a+q+ISOR5Yw39owdr2Bgf0qcoLvNx35uf88NnA820umSk8tAVpyrMRaTJ8Eag13RHvBG3huzoizfv5RsPfcAj760nNTlBzbREpEnyxpJLTULQC/1wSTn3vbmGpz/eRKf0Zjx93VBG9mobguJERELPu4EeAmU+P7OWb+ea07rwCzXTEpEmTglVzf6iUv7+0SZuHdWDVs2TePtnI2mZov4rItL0KdAr+c/y7dz12kr2FZVyem4Gw7pnKMxFJGoo0IGdB4r5zWsreWPll/Tv1JKnrxtC/05qpiUi0SX6A/2ehgfvzc99ytKCQu4Y04cbzuxGgpppiUgUiv5AP0EF+4po1TyJFskJ3HNJf1IS48lt2yLSZYmInDBvTkVH3Hbch/x+x1MfbeT8++fy57fWANC/U7rCXESinjdn6Me5Bj1/Z6CZ1qLN+xjZqy3Xn9EtzIWJiDSe6A70eqyfz1y6jZ+/uJTmyfFM/u5Axp/aWc20RMRTojvQg+D3O+LijIFZ6Ywd0IE7v9GPtmnJkS5LRCTkoncN/XgdFlPbA4FmWn/8z+fc9Ozir5ppPXD5qQpzEfGs6A30jx6oefsv1rJg417GPvgBU+asp3XzJMp8NTTvEhHxGE8tuZQDv311Bc/M30x2m2Y8e/0wzuiZGemyRETCIjoD/Tgvhh66fRdvPTCX60Z04+cX9KJ5UnSenojIifBE4rmKj1bNk3jnZ2erK6KIxKSg1tDNbIyZrTGzfDObVMPjZmYPVTy+zMwGhb7UCn/IAr6+pYWr+M/RW04ozEUkVtUZ6GYWDzwCXAj0A64ws37VdrsQ6FnxMQF4LMR1fq30IA6ofAW5M0i4p7DRnlJEJBoEM0MfCuQ75zY450qB6cC4avuMA6a5gPlAKzPrGOJaYdr4KsOjs/TovVRHRCR0gsnCzsDWSuOCim313Qczm2Bmi8xs0a5du+pbK2yZFzjO0eMd/VyzcxGRoAK9pvfHV7+wO5h9cM5Ndc7lOefy2rY9gXtz5pxe/68REYkRwQR6AZBdaZwFbDuBfRrumleg+6iq2zQ7FxEBgrtscSHQ08y6AV8AlwNXVttnJnCLmU0HhgGFzrntIa30qGteaZTDiohEuzoD3TlXbma3AG8C8cCTzrmVZnZTxeNTgFnAWCAfKAKubbySRUSkJkFdtO2cm0UgtCtvm1LpcwfcHNrSRESkPnTFn4iIRyjQRUQ8QoEuIuIRCnQREY+wwOuZEXhis13A5hP88kxgdwjLiQY659igc44NDTnnLs65Gt+ZGbFAbwgzW+Scy4t0HeGkc44NOufY0FjnrCUXERGPUKCLiHhEtAb61EgXEAE659igc44NjXLOUbmGLiIix4rWGbqIiFSjQBcR8YgmHehN6ubUYRLEOV9Vca7LzGyemQ2MRJ2hVNc5V9pviJn5zOzScNbXGII5ZzM728yWmNlKM5sT7hpDLYif7XQz+5eZLa0456ju2mpmT5rZTjNbcZzHQ59fzrkm+UGgVe96oDuQBCwF+lXbZyzwHwJ3TDoN+CTSdYfhnE8HWld8fmEsnHOl/d4l0PXz0kjXHYbvcytgFZBTMW4X6brDcM6/Av634vO2wF4gKdK1N+CczwIGASuO83jI86spz9Cbzs2pw6fOc3bOzXPO7asYzidwd6hoFsz3GeBW4GVgZziLayTBnPOVwAzn3BYA51y0n3cw5+yANDMzoAWBQC8Pb5mh45ybS+Acjifk+dWUAz1kN6eOIvU9n+sJ/IaPZnWes5l1BsYDU/CGYL7PvYDWZva+mS02s2vCVl3jCOacHwb6Erh95XLgJ845f3jKi4iQ51dQN7iIkJDdnDqKBH0+ZnYOgUA/o1EranzBnPMDwB3OOV9g8hb1gjnnBGAwcC7QDPjYzOY759Y2dnGNJJhzvgBYAowCcoHZZvaBc+5AI9cWKSHPr6Yc6E3n5tThE9T5mNnJwOPAhc65PWGqrbEEc855wPSKMM8ExppZuXPu1bBUGHrB/mzvds4dBg6b2VxgIBCtgR7MOV8L/NEFFpjzzWwj0AdYEJ4Swy7k+dWUl1y+ujm1mSURuDn1zGr7zASuqXi1+DQa8+bU4VHnOZtZDjADuDqKZ2uV1XnOzrluzrmuzrmuwEvAj6I4zCG4n+3XgDPNLMHMmhO4+frqMNcZSsGc8xYCf5FgZu2B3sCGsFYZXiHPryY7Q3cxeHPqIM/5N0AG8GjFjLXcRXGnuiDP2VOCOWfn3GozewNYBviBx51zNV7+Fg2C/D7fCzxlZssJLEfc4ZyL2ra6ZvY8cDaQaWYFwN1AIjRefumt/yIiHtGUl1xERKQeFOgiIh6hQBcR8QgFuoiIRyjQRUQ8QoEuIuIRCnQREY/4fz61ObICG2GJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr_probs = lr_model.predict_proba(X_test)\n",
    "# keep probabilities for the positive outcome only\n",
    "probs = lr_probs[:, 1]\n",
    "# calculate AUC\n",
    "auc = roc_auc_score(y_test, probs)\n",
    "print('AUC: %.3f' % auc)\n",
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, probs)\n",
    "# plot no skill\n",
    "pyplot.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "42378362",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD4CAYAAAAn3bdmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAyt0lEQVR4nO3dd3xVRdrA8d9z03uBBEISeug9SBFFqhRdxYKirugurwjq2ldRREWXXTuoKIrKoq6KbV0RQaWoSA2hVyH0kEAgEEr6Teb9457EG0iFJHAvz5fP+XDunDlnzoTwZDJnZo4YY1BKKeUabOf7BpRSSlWeBm2llHIhGrSVUsqFaNBWSikXokFbKaVciGdNFyADY3R4ijpD9g/bz/ctqAuQr4e/nOs1qhJzzPzkcy6vtmlLWymlXEiNt7SVUqpWics1nqtEg7ZSyr14aNBWSinX4d4xW4O2UsrNaPeIUkq5EDcfXqFBWynlXrSlrZRSLsS9Y7YGbaWUm9HRI0op5UK0e0QppVyIe8dsDdpKKTdjc++orUFbKeVe3Dtma9BWSrkZD/ceqK1BWynlXrSlrZRSLkRHjyillAtx75itQVsp5WZ09IhSSrkQ947Z7r4ellLqouMhld8qICIzRCRNRDY5pX0uIuusbY+IrLPSG4tIttOxd5zOiReRjSKSJCJviDg63kXEx7pekoisFJHGFd2TBm2llHsRqfxWsZnAYOcEY8zNxphOxphOwNfAf50O7yw6ZowZ45Q+DRgNxFlb0TVHAceMMc2BycCLFd2QBm2llHuRKmwVMMYsBo6WWoyjtXwT8Fm5tyMSBQQbY5YbYwzwETDMOnwt8KG1/xXQv6gVXhYN2kop91KFlraIjBaRRKdtdBVKuhw4ZIzZ4ZTWRETWisivInK5lRYNJDvlSbbSio7tBzDG2IHjQJ3yCtUHkUop91KFpqgxZjow/SxLuoWSrexUoKExJl1E4oH/iUhbSm/TG+vv8o6VSoO2Usq91MKQPxHxBK4H4ovSjDG5QK61v1pEdgItcLSsY5xOjwFSrP1kIBZItq4ZQhndMUW0e0Qp5V5sUvnt7A0Athljirs9RCRCRDys/aY4HjjuMsakAidFpIfVXz0S+NY6bTZwh7V/I7DI6vcuu3rnctdKKXXBqcbRIyLyGbAcaCkiySIyyjo0gjMfQPYGNojIehwPFccYY4pazWOB94EkYCcwz0r/AKgjIknAw8C4iu5Ju0eUUu6lGntHjDG3lJF+ZylpX+MYAlha/kSgXSnpOcDwqtyTBm2llFupYMScy9OgrZRyKxq0lVLKhXjoglFKKeU6tKWtlFIuRIO2Ukq5EA3aSinlQtw8ZmvQVkq5F21pK6WUC7GJe0/01qCtlHIr2tK+CIUHhbLw5c8BqB8WQUFhIYePpwPQ7b6rybfnn3MZP7/yJYF+/lxy71UAxLfowCujJ9D30SrNaFW1qHO7eOLimhd/njx1MtHRDUrN2yP+UlasXnZO5U148mkSV60mKDAQsdl4csI4OnbqeE7XvBi4eczWoF2aoycz6DxmEADP3P4wp7IzefWrd4uPe9g8KCgsOOdyIkPrMviSvvyw6udzvpaqeT4+Pnzxzee1WubDjz7IwEEDWbZ0Oc8/O4mv/vdFrZbvimxuHrU1aFfSv//+GkdPZNC5eTvWJG3kZFZmiWC+cfoCrp5wJ3sPJXNb/+u5f9hf8fbyYuXWtdzz5pMUFhaecc2Xv3iHp267/4ygbbPZeGHUk/Tp2AMfLx/emj2T6d9/gogw9b5/cEWHHuw+uB+bzcaMHz7n69++r5WvgSopKzOLB+57iBMnTmC327nv/nvo279viTyHDx/msYcfJ/NUJvaCAp56+km6dO3CsqXLmTZ1Gnl5+cTGxvDcpIn4B/iXWVZ81y7s37cfgI9mfsz//utY2fP6G6/jzyNvIysrm8cefoxDhw5RUFDI6LF3MXjIoJqr/AVMu0dUsRYxTRnw+AgKCwt55vaHS83TqmFzbr7iT/R6cBj2Ajtv/W0St/W7jo8XnLn41/Ktq7mu12D6dLyUk9mnitNHDb6F45kn6Hbf1Xh7ebN0yjf8tHox8XEdaFw/lvajBxAZWpetH/zMjB9qt+V3McvNzeWm624GoEFMNK9MfonJb75KYGAgx44d4/YRd9CnX58SQWPunHlc2utS7hrzfxQUFJCTk8OxY8d47533ePeDd/H392PG+//mow8/Zsw9d5dZ9q8/L6Z5XHO2bN7Ct9/M5j+zPgZjuG3E7cR3jedAcjIRkRFMfedNAE6ePFmzX4wLmE2nsasiXy6eU2qL2Vn/zpcR36I9q95ytH79vH1Jy0gvM/8/Pn2dp267n8ff/2dx2pXxvenQtDU39nb0d4f4BxEX3YTL2l3Cl4vnYIzh0LHD/Lx+eTXUSlXW6d0j+fn5vDFlKmsS12ATIS0tjfQj6dSNqFucp137tjwzfiJ2u52+/fvSqnVLEletZtfO3dx5253F1+nQqUOpZb72yhTee+d9wsLDePYfz5CwIoF+A/ri7+8HQP+B/Vizeg29Lu/Fqy9PZvKrr3PFFZfTpWuXmvtCXOC0pa2KZeZkF+/bC+zYbH8MLfL19gVAED786SuenPFCpa7587plPH/n3+nR+o//ZCLC396awE+Jv5bIe1X3/udy+6qazZ0zj2NHj/HZl5/g5eXFkAFDyc3LK5Envms8Mz5+n99+XcL4cU9x519HEhQcTI9Lu/PiKxV/jxT1aRdZuXxlqfkaN27ErC8/5bfFS3h9ypv0vLRHuS13d+buQdu9BzTWoD2HkunS3LGmeefm7WhSPxaAhWuXcGPvq4gIdbxQOSwolIaR0WVeB2DSp2/y2E1jiz//mPgrY6++HU8Px8/UuOgm+Pv6sWRTAjdcNhQRITK0Ln069KyJqqlKOnXqFOHhYXh5eZGwchUpKaln5Ek5kEJ4eDg3DL+e624YxtYt2+jQsT3r1qxn3959AGRnZ7Nnz95KlRnftQs/L/yF7OxssrKyWbTgZ7rEdyEtLQ1fP1+uvuYq7rhzJNu2bKvWuroScbxlvVKbK9KW9ln6+re5jBx4A2vf+ZFVv69n+4FdAGzdt4On/v0SP73wKTaxkW/P596pT7Ev7UCZ15qXsIjDTl0o78/7lMb1Y1gz7QcEOHz8KMOeGcXXv82lf+fL2PTeQrYn72LltrUczzxR01VVZRh69RDuv+cBbhl+Ky1btaRJ0yZn5ElclcjMGR/h6emJv78//3jhecLDw3nunxMZ9/cnyMtzDB+97/57aNy4UYVltm7TmmuG/Ynbbr4dcDyIbN2mFUuXLGPyK1OwieDp5cn4p5+s3sq6EFcNxpUlFbxD8twLGBhTswVcZAJ8/cnMySI8KJSEqXPo9eB1HDp2+HzfVpVl/7D9fN+CugD5evifc8SNer53pWNO6oTFLhfhtaXtYub840NCA4Px9vTi+U9ed8mArVRNcn7W5I40aLsYnTGpVPmqc3KNiMwArgbSjDHtrLRngbuAohbTk8aYudaxJ4BRQAFwvzHmRys9HpgJ+AFzgQeMMUZEfICPgHggHbjZGLOn3PpVW+2UUuoCIFL5rRJmAoNLSZ9sjOlkbUUBuw0wAmhrnfO2iHhY+acBo4E4ayu65ijgmDGmOTAZeLGiG9KWdhV98MgrXN19AGkZR2g/egAAHZq25p0HXiDQL4A9B/dz2wt/42TWH5NlYiMasOWDn3n2o9eKZ1DedMWfGH/r/XjYbHy/chGPvz8JgNfGPEPfTpcC4O/jR2RoHcKua1vLtVTVaciAofgHBOBhs+Hh6cFnX37Ktq2/84+Jk8jLzcXD04MnJzxJ+w7tWL5sBa+/9gb5+fl4eXnx0KMP0r1Ht/NdBZdSnQ8ijTGLRaRxJbNfC8wyxuQCu0UkCegmInuAYGPMcuv+PgKGAfOsc561zv8KmCoiYsp52KhBu4pm/vQlU7+dyUePTSlOe//hl3l0+j9YvGEFfxl0M38fPoanP3yl+Pjksc8yz2mqenhQKC+Pfor4e4Zw5PhRZv59Mv0692LR2qU8/M7E4nz3XfsXOjfXgO0O3p85nbCwsOLPk1+dwph7RnNZ78v47dffmPLqFD748H1CQ0N54+0pREZGsmNHEmPvuocFv/x0Hu/c9QiVD9oiMhpHC7jIdGPM9Eqcep+IjAQSgUeMMceAaGCFU55kKy3f2j89Hevv/QDGGLuIHAfqAEfKKrjC7hERaSUij4vIGyLyurXfuhKVcku/bVzJ0ZMZJdJaxjRj8QbHv9X8NYu54fKhxceuvXQQu1L3sXnPH6MlmkY1YnvyLo4cPwrAgrVLuOGyoZzulr7X8tnP39ZALdT5JiKcyswEHOO9IyIjAGjdphWRkZEANG/ejLzcPPJOm7CjyleVcdrGmOnGmK5OW2UC9jSgGdAJSAVeLSq6lLymnPTyzilTuUFbRB4HZlkXTgBWWfufici48s69mGza8zvX9LwSgOG9ryY2wrFcp7+vH4/ffA8TP36tRP6klD20im1Oo3oxeNg8GHbpoOJzijSMjKZJ/VgWrVtaO5VQNUeEMf93DyNuvJWvvnCsQfPYuEeZ/PIUruw3mFdfnsz9D/7tjNMW/LSAVq1b4u3tXdt37NJsNqn0djaMMYeMMQXGmELgPaCo/yoZiHXKGgOkWOkxpaSXOEdEPIEQ4Gh55VfUPTIKaGuMKbGAtIi8BmwGSp2HW+JXjlahEBNQQTGu7a+vPsIb9z7H039+kNnL55Nnrbc9ceQjTP76PTJzskrkzzh1nLFvPMHn46dRaApZtiWRpvVLTqwY0fdavvptboVrnagL34ef/JvIyEjS048y5v/G0KRpY+b/uIC/j3uEAVcO4Md5P/HshIlMn/HH8r9JO3Yy5bU3eOe9t8/jnbummp5cIyJRxpii6a/XAZus/dnAp1Z8bIDjgWOCMaZARE6KSA9gJTASeNPpnDuA5cCNwKLy+rOh4qBdaBV++hzbKOtYqaxfMabDxTG55vf9Oxk07jbAMeW8aI2Q7q06c+PlV/HSXeMJDQymsNCQk5/LW9/OZM6KBcxZsQCAu4beRkFByS/niD7XcO+b42u3IqpGFHV31KkTTr/+/di0YTPffTuHx598DIArBw9k4tPPFec/dPAQD93/MP/41/PENowt9ZqqbNUZtEXkM6APUFdEkoFngD4i0glHN8Ye4G4AY8xmEfkC2ALYgXuNMUUL74/ljyF/86wN4APgY+uh5VEco0/KVVHQfhBYKCI7sDrLgYZAc+C+ii5+sYgIrcPhjHREhKdue4B35nwMQO+HbyjOU/Qyhbe+nVninNDAEO65ZiQ3PT+mOG+LmKaEBYawfMvqWq2Hqn5ZWdkYU0hAQABZWdksX7acu8eOJiIygsRVq7mkW1cSViTQsFFDAE6cOMl9Y//GAw/9jc5dOp3fm3dR1Tx65JZSkj8oJ/8kYFIp6YlAu1LSc4AqTb4oN2gbY34QkRY4+myicfRnJwOrnH6CXFQ+fXIqfTr0pG5IOPs/XcUzH71KoF8A915zBwD/XTKPf/9Y8RrXr98zkY5N2wDw3H+msOPA7uJjt/QdxqxfZtdMBVStOpqezkP3O9Zet9sLGHrVEHpd3gs/f39e+tfLFBTY8fb24emJTwEw69NZ7Nu3n+nT3mP6tPcAmPb+NOrUCT9vdXA1br70iK49os4PXXtElaY61h5p/frQSsecrQ/MdbkQr+O0lVJuxd1X+dOgrZRyK24es3Xtkaqw2WysmfYD3z0/E4COzdqw/I3ZjjW13/qeS1p2KvW83R8vZ8P0BcX5ijxz+8Mkf5bI2nd+ZO07PzKkWz8ALm3blfXvzidh6hyaNWgMQEhAMD/86z81WT11Dp4e/yx9LuvH9dfcWG6+TRs307ldPPN/nF+cNmTAUG64djg3XXcztwy/tTh98quvc+Owmxg/7qnitO9mz+GTjz+t/gq4EX0Jgir2wHWj2LoviWD/QABeums8Ez+ezA+rfmZIt368dNf4Mlfh6/vocNJPHDsjffLX7xWvR1LkkRvv5obnRtO4Xgxj/3Q7j777PBP+/AD//Gxq9VdKVYtrr/sTt9x2M+PHTSgzT0FBAVNee51Le535xqHTp7mfPHmS9WvX89X/vuCJvz/Jju07iG0Yy+xvvuPt6fp9UB5XDcaVpS3tSoquG8VV3fvz/rw/WjnGmOIAHhIQREr6oWopK9+ej5+3L/4+fuTb7TSNakR0nfrFU+XVhSe+azzBISHl5vnsk1kMGNif8EqMBLHZbOTn52OMISc3F09PT2bO+JBb/zwCLy+v6rptt+TuLW0N2pU0ZeyzPPbeJAoL/3gw/eC0Z3l59FPs+ySBV0ZP4IkP/lXqucYYfnrhUxLfmstdQ28rcey+a+9k/bvz+eCRVwgNdPyn/9esqUx/6EUevP7/mPrtTCb95TEmOC1ApVzPoUNpLFqwiOE3l9J9Uso094CAAAZc2Z+brx9BdHQDAoMC2bxpC337963lO3c9NT2N/XzT7pFKuKp7f9IyjrBmx0aucHqZ7tirR/LQtIn8d8lchve+mg8eeYWBj585Fr/XQ9eRmn6IiNA6zH/hM7btT+K3jSuZ9t1HPP/JFIwxPH/n33n17gmMevVR1u/cQs/7rwHg8vbdSUk/hACzxr9Nvt3OI+8+R1pGmYuAqQvQy/96mQcfeQAPD48zjpU2zT2+azx/GXUnfxl1JwDPTpjIPfeN5b9f/ZflS1cQ1zKO0WPuquVauAgXbUFXlra0K6FX20u4pueV7P54ObPGv0W/Tr34+PE3uOPKG/nvkrkAfLl4Dt3KeBCZanWbHM5I55ulPxTnS8s4QmFhIcYY3pv7aannP3Xb/Tz/yes8c/vDPPPRq/xn4X+5/7q/1kQ1VQ3avHkLjz8yjiEDhjL/xwVMev5fLFrgWK63tGnuzrZab1Zv1LgR3307h5cnv0TSjiT2VvIN7hcb7R5RPDnjBWJvvYQmt/dkxKR7WbRuKbe/eD8p6YeKW979OvcqMauxiL+vH4F+AcX7V8b3ZtOe3wGoHx5ZnO+6XoOL04vcceVwvl+5iIxTx/H38aOwsJBCU4i/j19NVVXVkHnzv2fegrnMWzCXgYMGMH7CE/Qb0JesrGwyrSVai6a5N49rVuLct958m3v+Nha73V68gJjNZiMnJ6fW6+EKqvnNNRcc7R45B3e99hiv3zMRTw9PcvJyGT3lcQCi6tTj/Ydf5qrxI6kXGsE3z74PgKeHB5/+/D9+TPwFcIw+6dSsLcYY9hzaz91T/ljt1s/HlzsGDufKcY4hYK99PZ2vn3mPvPw8bvnnvbVbUVWhxx8dR2LCajIyMhjYdxBj7xuDPd8OwE0jyl5aoqxp7kUWLfiZdu3aFrfGO3TswA3XDqdFizhatmpZgzVyXa7agq4sncauzgudxq5KUx3T2Lt+cEOlY07iqK9dLsJrS1sp5VZcdVRIZWnQVkq5FXfvHtGgrZRyKxq0lVLKhWjQVkopF6JBWymlXIg+iFRKKRfi7i1tnRGplHIr1TmNXURmiEiaiGxySntZRLaJyAYR+UZEQq30xiKSLSLrrO0dp3PiRWSjiCSJyBtiFS4iPiLyuZW+UkQaV3RPGrSVUm6lmqexzwQGn5Y2H2hnjOkAbAeecDq20xjTydrGOKVPA0YDcdZWdM1RwDFjTHNgMvBiRTekQVsp5Vaqs6VtjFkMHD0t7SdjjN36uAKIqeB+ooBgY8xy45iC/hEwzDp8LfChtf8V0F8quDEN2kop91KFpraIjBaRRKdtdBVL+yswz+lzExFZKyK/isjlVlo0kOyUJ9lKKzq2H8D6QXAcqFNegfogUinlVjyqMHrEGDMdmH425YjIeMAOfGIlpQINjTHpIhIP/E9E2gKl3VDR+ijlHSuVBm2llFupjdEjInIHcDXQ3+rywBiTC+Ra+6tFZCfQAkfL2rkLJQZIsfaTgVggWUQ8gRBO6445nXaPKKXcik2k0tvZEJHBwOPANcaYLKf0CBHxsPab4njguMsYkwqcFJEeVn/1SOBb67TZwB3W/o3AIlPB0qva0lZKuZXqbGmLyGdAH6CuiCQDz+AYLeIDzLfKWmGNFOkNPCcidqAAGGOMKWo1j8UxEsUPRx94UT/4B8DHIpKEo4U9oqJ70qCtlHIr1dl9YIw586WvjkBbWt6vga/LOJYItCslPQco+y0ZpdCgrZRyKx429+711aCtlHIrZ9tX7So0aCul3Iq7rz2iQVsp5Vbcu3NEg7ZSys1o94hSSrkQ7R5RSikX4qFBWymlXId2jyillAvRoK2UUi5E+7SVUsqFaEtbKaVciHuHbA3aSik346lrjyillOvQPm2llHIh2qetlFIuxL1DtgZtpZSb0Za2Ukq5EH0JglJKuRD3DtnuXz+l1EVGRCq9VeJaM0QkTUQ2OaWFi8h8Edlh/R3mdOwJEUkSkd9FZJBTeryIbLSOvWG9lR0R8RGRz630lSLSuKJ70qCtlHIrNpFKb5UwExh8Wto4YKExJg5YaH1GRNrgeJt6W+uct0XEwzpnGjAaiLO2omuOAo4ZY5oDk4EXK6xfZe5aKaVcRXUGbWPMYuDoacnXAh9a+x8Cw5zSZxljco0xu4EkoJuIRAHBxpjlxhgDfHTaOUXX+groLxX8ClDjfdr3TxhR00UoF/RcwqTzfQvqAvTPnuf+fVELk2vqGWNSAYwxqSISaaVHAyuc8iVbafnW/unpRefst65lF5HjQB3gSFmF64NIpZRb8ZDKdyCIyGgc3RZFphtjpp9l0aX9tDDlpJd3Tpk0aCul3EpVxmlbAbqqQfqQiERZrewoIM1KTwZinfLFAClWekwp6c7nJIuIJxDCmd0xJWiftlLKrUgV/pyl2cAd1v4dwLdO6SOsESFNcDxwTLC6Uk6KSA+rv3rkaecUXetGYJHV710mbWkrpdxKdfZpi8hnQB+grogkA88ALwBfiMgoYB8wHMAYs1lEvgC2AHbgXmNMgXWpsThGovgB86wN4APgYxFJwtHCrvAhoAZtpZRbqc5p7MaYW8o41L+M/JOAM56mGmMSgXalpOdgBf3K0qCtlHIr4ua9vhq0lVJuRdceUUopF3IODxhdggZtpZRb0aVZlVLKhejrxpRSyoXY9EGkUkq5Dps+iFRKKddh0weRSinlOrRPWymlXIiOHlFKKRei47SVUsqF2KqwnrYr0qCtlHIrGrSVUsqFaJ+2Ukq5EO3TVkopF6ItbaWUciGifdpKKeU6tHtEKaVciL4EQSmlXIi7rz3i3j+SlFIXHRGp9FbBdVqKyDqn7YSIPCgiz4rIAaf0oU7nPCEiSSLyu4gMckqPF5GN1rE35BwWSNGgrZRyKyK2Sm/lMcb8bozpZIzpBMQDWcA31uHJRceMMXMd5UobYATQFhgMvC0iHlb+acBoIM7aBp9t/TRoK6Xcig2p9FYF/YGdxpi95eS5FphljMk1xuwGkoBuIhIFBBtjlhtjDPARMOwsq6dBWynlXmxiq/QmIqNFJNFpG13GZUcAnzl9vk9ENojIDBEJs9Kigf1OeZKttGhr//T0s6vf2Z6olFIXoqr0aRtjphtjujpt00u5njdwDfCllTQNaAZ0AlKBV4uylnI7ppz0s6KjR5RSbqUGRo8MAdYYYw4BFP0NICLvAXOsj8lArNN5MUCKlR5TSvpZ0Za2UsqtVNeDSCe34NQ1YvVRF7kO2GTtzwZGiIiPiDTB8cAxwRiTCpwUkR7WqJGRwLdnWz9taSul3Ep1zogUEX9gIHC3U/JLItIJRxfHnqJjxpjNIvIFsAWwA/caYwqsc8YCMwE/YJ61nRUN2kopt1Kd74g0xmQBdU5Lu72c/JOASaWkJwLtquOeNGgrpdyKvgRBKaVciLtPY9egrZRyK9XZPXIh0qCtlHIr4uaD4jRon+aH0d8RFB1c/LnzvZfgX9e/1Lzz75vLwKlDSz1WWRtmrCV96xGu+Gc/bF4e5J3MZdmk3+jzwoBzuq6qGbmncvntxcUA5BzPQWyCT5APAP2e6Y/N89wDxq//+oWc4zl4eHng6eNJ/KiuBEUFnfN1Lxba0r7IeHh70OuZK2q1TLFB8tL9NOzTuFbLVVXnE+jDgOcHArDlm814+nrSYkjL4uOFBYXYPM49cHe7uxthTcLZ9csuNn6+gUsf7HXO17xYeOiDyIubPcfOmrdWYc/Ko7DAEDesFfU61S+RJycjh/XTV2PPtmMKDW1ua094izoc2ZzGjtnbMfYC/CICaH9nJzx9z/ySN+rflD3zdxFzecMzju3+MYnUVSkU2gup1zmKuGsdASJpznZSVybjG+aHV6A3IY1CaTKoWc18EVS5Et9bhVegN8f3ZhDaKBRPX88SwXz++J+49MFeBEQEsG/ZXpLmJ1FoLyS8WTidR3ZBbGW3DOu2qEvSTzswxrDx840c2ngQBFr9qTWx3WPJzsgm4e2V5GfnYwoNnUd2pm7LiNqq+gVJ31xzkSnIK2DpxF8B8KvrT6cx8XS5pyuefl7kncxlxb+WENmxXolfwVITDlC3bQTNrmqBKTQU5BWQdzKXnd/v4JKHe+Dp48mueUnsmb+L5n9qcUaZfnX8CIsLJ2VFMpEd6hWnH9mcRuahTHqOvxwMrJmawNHt6Xh4e3BodSqXTrgCU2hY9vxiQhqF1vjXRpXt1MGTXP5Yb8QmbPlmc6l5TqScYP/KZPqM74vN08baj9awb/k+GvVqVOZ1U9elEhITQkriAY7vz2DA8wPJPZnLookLqduyLvuX76deu3q0uqY1ptBgz7XXVBVdhnaPXGRO7x4ptBey/ZttHN2ejoiQk5FD3olcfEJ8i/OENA5l08x1FBYY6nWqT3DDENJ+T+dU6klWvrDUcZ2CQkKbhp1RXpGmQ+NYMzWBiPZOQXvLYY5sOcyy5xx9qAW5drIOZWLPtRPZqT4e3o6leiOcAr06P2IuiSm3xQyQtiWNjL3HWDRxIQAF+QXF/eGnS3g3AQ8vD/zrBtDpz53Y8eN2YrvHIjbBN8SXiJYRHNt9jLCmYaz+IJHCgkIadIkmVH9464PIi13KygPknczj0qd6Y/O08cu4BRTkF5bIE96iDt0e68XhDYfYMGMtTa5shleAF3VaR9BpdHylygmIDCA4NoSDiU7ryBhoOqQ5Da9oXCLvnvk7z7Vaqpp5+PzxX0k8bBinNdwK8q2ZzAYa9WpEu+HtK7xeUZ92sTLWhItoGcEVT/Th4PpUVr2XQIshLcttuV8M3L2l7d4/kqqBPTsf7yBvbJ420rcdISc9+4w82elZeAd5E9u7ETGXNeTEvuOENg0jY+dRMtMyAUcrOfPgqXLLajo0jt0//RGQ67aN4MDS/dhzHL/y5hzLJvdELmHN63B4w0EK8guw59g5vPFQWZdU50FAXX8y9hwD4NieY2QednwPRLaJJDnxADkncgDIO5VH5pHMSl2zbssI9ickYwoNuSdyObL9COFNwsk8kolPsA9N+jSl8eVNisu9mNXQSxAuGNrSrkCD7tGsnrqKZf9YTHBsCAH1A8/Ic/T3dHb/uBPxEDx9PGn/1854B/nQ/s7OrJ++mkK7o2XeYlirUs8vEhQdREjDEI7vOw5A3baRnEo9xYoXlgCO1lzHUZ0JaRJKRMf6LJ34K351/AhpFIqnn/5TXiiiu8awd+leFkyYT1iTcILqO4brBUcH0/b6tix5+TeMAZuH0On2zgTUDajwmg3iG5CelM6CCfNBoN1N7fEN9WXvkj1sn7cd8bDh6evBJXd1q+nqXfDcfRq7GHPWa3FXygOLH63ZAi5S9hw7nr6eFOTaWfnyMtre3sGlHkYGeJXel6subv/sOemcm7+LUuZVOub0azDE5Zrb2jxzUZs/3sCplJMU2gto0DPWpQK2UjVJH0SqC1LHu7qc71tQ6oJkc/MHkRq0q2jjzHUc3nAI7yAfLpvYB4C8zDzWv7ua7PRs/Or40enueLwCvDmy5TDbv95aPEuu5Y1tqNO6LvYcOytfXFp8zZyMbBp0j6H1iHZs/XwTR7elAxSP9x7wxpDzUFN1Lnb8tIM9v+7GGGhyRRPiBsWRsTeDtR+uoSC/APGw0XlkZ8KbOkaIbJuzjT2LdyM2oeNtnajfvj72XDsr31rBqbRMxCZEdYqi/U0Vjzy52OnkGlVC9KWxNOzbmI0z1hWn7Z6XRJ3WdWk6JI5d83awa14SLW9sg3egN13+1g3fUF9OHjhB4pSV9H15IJ6+niXGgi97fjH1ujjeYNT65j/WSd+7cDcn9h+vtbqp6nE8+Th7ft1N36f7YfO0seTVJdTvWJ+NX2yg9bDW1O8QRer6VDZ+voErnujDiQMnSF65n4GTriQnI4ffXlrMoBcHAxA3pAWRrSMptBey+KVfObghlfodoiq4g4ubDvlTJYS3qINXgHeJtEPrDtKgp+N9ng16xnJo3UEAghuG4BvqmIQT2CCIwvwCCovG7FoyD50i72QuYXHhnC511QGiukXXRDVUDTqZcpLwZuF4+nhi87AR0bIuKWtSQIT8bMfwTXt2Pn5hfgCkrE0hpnssHl4eBEQEEFAvkKO7juLp40lk60gAbJ42QhuFkX30zCGnqiSb2Cq9uSLXvOsLTN6J3OLg7BvqS97JvDPyHFqTSnDDEGxeHiXSUxNSqH9JgzNaB9npWWQfyaJOq7o1d+OqRgTHBHPk9yPknsrFnmvn4IaDZKdn0fHWjmz8fANzH/6eDbM20PZGx29V2cey8Q/3Kz7fP8yP7GMlg3NeZh6p61KJaBNZq3VxRbYq/HFFZ33XIvKXco6NFpFEEUncNHvD2RbhNk4eOMnvX2+l7Z87nHGsrNZ0akIK9bpEVTg1Wl14ghsE02JoS5a8/BtLX11CSGwo4iHsWrSLjrd0ZOhrV9Hx1o6snrHacUIpA9Sc/9ULCwpJeGclzQc0JzCy7HH+ykFEKr25onP5UTOxrAPGmOnGmK7GmK7trjkzULkb72AfcjIcs9xyMnLwDvqj+yTnaDZr315Fh792xj+y5CSKE/uPYwpMqcP1tGvEtTW5ogn9Jw7giif74B3oRWC9IPYu3UODro5/0+hLYji26ygAfuF+ZDl1e2Qdy8Y37I+W95qZawisF0TcoLjarYSLkir8qfBaIntEZKOIrBORRCstXETmi8gO6+8wp/xPiEiSiPwuIoOc0uOt6ySJyBtyDj8xyg3aIrKhjG0joKsUWSI71idl+X4AUpbvL166NT8rn9VvJtDi+laENS+lzzohpdTAfOrgKfKz8gltVvYCU+rCVjRVPSs9iwOJKcT2iMUv1I8j2w4DcHhrGoH1HK3mBp2jSF65n4L8AjIPZ3Lq0KniUSWbv95EflY+HW/teH4q4oJqoKXd1xjTyRjT1fo8DlhojIkDFlqfEZE2wAigLTAYeFtEivpDpwGjgThrG3y29ato9Eg9YBBw+oIGAiw720Jd2brpqzm2PZ28U3n8/Pf5xF3TkqZDmrPu3dUkL9mPb7gfncY4Fonat2g3WWmZ7Jyzg51zdgDQ9aEe+AQ7ZgMeTEwh/v4zpx2nJhwg6pJol/31TcGKqcvJO5WHzcNG55Gd8A7wpstf4ln/yTpMocHmZaPLXxzfJ8HRIcRcEsP8J39CPITOt3dCbELW0Sy2fbeNoKggFj6zAIBmA5rT5Iom57NqF7xa6Ku+Fuhj7X8I/AI8bqXPMsbkArtFJAnoJiJ7gGBjzHIAEfkIGAbMO5vCy53GLiIfAP82xiwp5dinxphbKypAp7Gr0ug0dlWa6pjGnnhkWaVjziURve7G0QIuMt0YM73og4jsxtFoNcC7xpjpIpJhjAl1ynPMGBMmIlOBFcaY/1jpH+AIzHuAF4wxA6z0y4HHjTFXn039ym1pG2NGlXOswoCtlFK1rSqTa6wAPb2cLL2MMSkiEgnMF5Ft5RZdShHlpJ8V1xzzopRSZajOPm1jTIr1dxrwDdANOCQiUVZZUUCalT0ZiHU6PQZIsdJjSkk/Kxq0lVJupbpGj4hIgIgEFe0DVwKbgNnAHVa2O4Bvrf3ZwAgR8RGRJjgeOCYYY1KBkyLSwxo1MtLpnCrTaeyVkH00m40z1pJ7PBcRiOndiMYDmrJj9u8k/7YP70DHEL8W17cq8bqwIr+MW4Cnr6fjp7uHcOlTvQHHkL/N/9lIQa4dvzr+dPy/znj6eXEs6Sib/7MBm6eNjqPjCYgMID8rn3Xvrqbrg931AeUFIis9i8T3VpFzPAdEaNKnCXFX/jEsb/u839n4+UaufvNPZb5WzBQaFj67EL8wX3o9dBkAyQnJbPnfFk6mnqDf0/2K32BzZMcR1n64Fg8vG93GdCewXiB5mXmsnLaSyx65TL8vLNW49kg94Bvr6+oJfGqM+UFEVgFfiMgoYB8wHMAYs1lEvgC2AHbgXmNM0RToscBMwA9HP/dZPYQsuhFVAbEJLYe3IaRRKPYcO8ueX0zdNo43Xjce0LRSb0Hv9khPvE/7j7vpw/W0Gt6G8JZ1SV6yj90/7iRuWCv2/LSTzmO7kp2ezf5f9tDqprbsnLOdZkOb63/MC4h4CO1HdCCscRj52fksenYh9drWIzg6mKz0LA5tTsO/jn+519jx0w6CGwSRn51fnBYcE0zPv/VkzczVJfP+sJ0e9/Ug60gWuxbtpMMtHdk2eyutrm6l3xdOqmt6ujFmF3DGWEtjTDrQv4xzJgGTSklPBNqdeUbVafdIJfiG+hZPgPH09SQwKrB4Ms25yDyUSViLOgDUaRPBwTWpgOMdg4X5hRTkOVaDy0rLJCcjh/CWOqX9QuIX6kdYY8dYei8/L4IaBBVPP9/w2foKV+TLOprFwfWpNO5dcghfcINggqKCzshv87BRmF9Q/H1xKu0U2ceyiWgVUU01cg/VObnmQqQt7SrKOpLFif3HCW0SyrGko+z9eTcHlu8npHEorYa3OWMxKXA8Ok6csgIQYq9oRGxvx4tXg6KDSFt/iHqd6nMwMYUca1Zc0yHN2fTRejy8Pejw185s+2oLcde2rMVaqqrKPJxJxt4MwpuFk7I2Bd8wP0IbhpZ7zoZP19P+5g7YnVrZ5Wl5VSvW/HsNHt4edB19CRtnbaDN9W2r4e7di7v/1qFBuwrsOXbWTUuk1c3t8PTzomGfxjS/ugUAO77dxrYvt9D+zk5nnNd93GX4hvqSeyKXxMkrCKgfSHiLOrS7oyNbZ21i53fbiexYD5un4xef4IYh9HzycgCObk/HJ8QXDKx7dzXiIbS6qW3xBB11/tlz7KyYupyOtzomxWz7biuXP9q73HNS16XgE+xDWOMwDm9NKzdvkdBGofR9uh8Ah38/XDzVfeXbKxAPGx1GdMA3xPfcKuMGXLUFXVnaPVJJhfZC1k5LJKp7NPWtta99gn0QmyA2IebyRhzfnVHquUUrAPoE+xDZuX5xvsCoIC55qCeXTuhNVLdo/CNKrk1ijGHn99tpfnUcSd9tp/k1LWjQI4a9C3fVWD1V1RTaC1k+dTmxPRsS3TWazLRMsg5nsWDCfOY9MpfsY9ksfGbBGd1p6TvSSV2byrxH5rJy2koObz1MwrsJlSrTGMO22VtpfU1rtv5vC62HtaFhz4YkzU+qiSq6HHdfMEpb2pVgjGHTh+sJjAqkyZV/PHTMycgpDshpa1MJjD6zH9Keawfj6Au359pJ33KYZlbrPPdELj7BPphCw87vdxB7RaMS5x5YlkxE+3p4BXg7+jFFEHG80Uadf8YYVs9IJDgqiBaDHf+mIbEhXP3mn4rzzHtkLv2e7X/G6JF2w9vTbrijz/vw1jS2/7CdbndX7k3qe5fspX7HKLyLvi9sgtigIM9eTTVzbe7e0tagXQkZSUdJWZFMYHQQSyf+CjiG96UmHODE/hMA+NX1L156NScjh00frqfrA93JO5HL2rcTATAFhUR1jyainWNN5NSEA+z7eQ8A9bpEEd3rj3H5Bbl2Upbvp+uDPQBoPLApa99JxOZh0/dDXiDSd6Szb9k+gmNCWDBhPgBtb2xHVMfS3yyTfSyb1f9ezWUPX1budQ+sPsD6/6wj92QuSycvJaRhKJc/6ugus+fa2bd0L5dZn+MGxbFi6nJsHja6je1ejbVzXa76coPKKnftkeqga4+o0ujaI6o01bH2yPbjmyodc1qEtHO5Zrm2tJVSbkW7R5RSyoW46gPGytKgrZRyMxq0lVLKZbj7g0gN2kopt6J92kop5UK0T1sppVyItrSVUsqFaNBWSikXot0jSinlQnT0iFJKuRDtHlFKKZeiQVsppVyGe4dsfQmCUsrNVNdLEEQkVkR+FpGtIrJZRB6w0p8VkQMiss7ahjqd84SIJInI7yIyyCk9XkQ2WsfekHN4WqotbaWUm6m2trYdeMQYs0ZEgoDVIjLfOjbZGPNKiVJF2gAjgLZAA2CBiLQwxhQA04DRwApgLjAYmHc2N6UtbaWUW6mut7EbY1KNMWus/ZPAViC6nFOuBWYZY3KNMbuBJKCbiEQBwcaY5cbxAoOPgGFnWz8N2kopt1KV7hERGS0iiU7b6DKu2RjoDKy0ku4TkQ0iMkNEwqy0aGC/02nJVlq0tX96+lnRoK2UumgZY6YbY7o6bdNPzyMigcDXwIPGmBM4ujqaAZ2AVODVoqylFVFO+lnRPm2llFupznHaIuKFI2B/Yoz5L4Ax5pDT8feAOdbHZCDW6fQYIMVKjykl/axoS1sp5Vaqq0/bGuHxAbDVGPOaU7rzm5uvAzZZ+7OBESLiIyJNgDggwRiTCpwUkR7WNUcC355t/bSlrZRyK9W49kgv4HZgo4iss9KeBG4RkU44ujj2AHcDGGM2i8gXwBYcI0/utUaOAIwFZgJ+OEaNnNXIEdCgrZRSpTLGLKH0/ui55ZwzCZhUSnoi0K467kuDtlLKrejaI0op5VI0aCullMtw75CtQVsp5Wb0JQhKKeVCtE9bKaVcigZtpZRyGe7ePaIzIpVSyoVoS1sp5Va0T1sppVyKBm2llHIZNjfv09agrZRyMxq0lVLKZbh3yNagrZRyO+4dtjVoK6XciruP09agrZRyK+4+5E8cb3RXtUFERpf24lB1cdPvC1UVOiOydo0+3zegLkj6faEqTYO2Ukq5EA3aSinlQjRo1y7tt1Sl0e8LVWn6IFIppVyItrSVUsqFaNBWSikXokG7lojIYBH5XUSSRGTc+b4fdf6JyAwRSRORTef7XpTr0KBdC0TEA3gLGAK0AW4RkTbn967UBWAmMPh834RyLRq0a0c3IMkYs8sYkwfMAq49z/ekzjNjzGLg6Pm+D+VaNGjXjmhgv9PnZCtNKaWqRIN27ShtBRsda6mUqjIN2rUjGYh1+hwDpJyne1FKuTAN2rVjFRAnIk1ExBsYAcw+z/eklHJBGrRrgTHGDtwH/AhsBb4wxmw+v3elzjcR+QxYDrQUkWQRGXW+70ld+HQau1JKuRBtaSullAvRoK2UUi5Eg7ZSSrkQDdpKKeVCNGgrpZQL0aCtlFIuRIO2Ukq5kP8He/Z1rg7ASL4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# review a confusion matrix\n",
    "\n",
    "lr_matrix = confusion_matrix(y_test, lr_pred) \n",
    "# print(lr_matrix)\n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in lr_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in lr_matrix.flatten()/np.sum(lr_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(lr_matrix, annot=labels, fmt='', cmap='Greens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "492ca2eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.65      0.97      0.78     20079\n",
      "         1.0       0.94      0.49      0.64     20079\n",
      "\n",
      "    accuracy                           0.73     40158\n",
      "   macro avg       0.80      0.73      0.71     40158\n",
      "weighted avg       0.80      0.73      0.71     40158\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "lr_report = classification_report(y_test, lr_pred) \n",
    "print(lr_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a00dd7e",
   "metadata": {},
   "source": [
    "## Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "667ce0d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9785347875890233"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nb_pred = nb_model.predict(X_test)\n",
    "nb_score = nb_model.score(X_test, y_test)\n",
    "nb_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "8d91ac4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.974\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAleElEQVR4nO3deXxU9b3/8dcne4AQIGGHEPZdFMLiBoqiiCj1aut2Vawt0mqr0lZo1dbben/33lpBrVaKVlHbilRQ0aKIooArBIUAIhB2BGVfQ0gy8/39MQGSGJIJTDIzZ97PxyOPme/MIfM5Bt75+j3nfI455xARkegXF+4CREQkNBToIiIeoUAXEfEIBbqIiEco0EVEPCIhXB+cmZnpsrOzw/XxIiJRacmSJbucc00rey9sgZ6dnU1ubm64Pl5EJCqZ2aaTvaclFxERj1Cgi4h4hAJdRMQjFOgiIh6hQBcR8YhqA93MnjWzHWa24iTvm5k9bmb5ZpZnZn1DX6aIiFQnmNMWpwJPAC+c5P3LgM6lXwOBp0ofpSpbFsHGhZB9PrQdAA+1hJICsDi4fFJgm1WvQ/dR5Z/njD7593ww/cTzVv1gzLyTb5s7NbjvKSKhVfHffghZMO1zzSwbeNM516uS9/4KfOCce6l0vBq4wDm3varvmZOT4zx9HnrZcD2m0zC48DcwfTTsLz2VNC4RnANXEtz37XcrdLgAnB9wpX/WwcwffXfbBi2g783ffX37Ulj7zolx50ug5ZnBfb6InDLf/q3E5U3HnB8SkuGWWTUOdTNb4pzLqey9UFxY1BrYUma8tfS17wS6mY0BxgBkZWWF4KOrUFmgltXhAuh6OfS9KRCIx8Kx4qPzB7avdhsHj/cDX+HJPzN/buCrLH9xzfZryXOBr2Ac+gYWPFzJGxV+ia99B9bOrWQ7EQkVB1jZf3u+osBMPYSz9FAEulXyWqXTfufcFGAKBGboIfjsylUX5gDrPwh8vfWrWisjeHGAP7hNB98LPa8Cs8DyDBZ4/kQlv7BPtuySOxXevOvEeORjWnYRqSWFxT4ee28tUxasZ3DqBp7h98T7iyE+KbDsEkKhCPStQNsy4zbAthB831Pz/9rUbPsmHUqXJUqDseJj2dA8/kiZcVzg+Rt3newTyktpDIV7y7922xx4ftTpraE/uD/4NfRj30Nr6CK1bsyLS1iwZiff79eG+y8fRvzu/hG9hn45cCcwgsDB0Medc9VWWWtr6MHMzssK1ez02EHNqvT+AVz9NEwZGljHbtAcfvB8yH+oIhJeh46WkBBnpCTG88m63ZT4/ZzfudJ+WjV2WmvoZvYScAGQaWZbgd8BiQDOucnAbAJhng8UALeGpOra1nFoaGen92+vPNQ7DIWbXy3/WlVnn4hIVJu/Zie/mbmc753Vil9d2o2zO2bU2WdXG+jOueured8Bd4Ssotrw4P66+Zz7qzyxR0Q8bF9BEX94cxUzPt9Kx6b1GdqtWZ3XELb2uSIiXvFR/i7umraUfQVF3HlhJ+4c2omUxPg6r8N7gZ6eBfs3lx+LiNSijAZJtG2SyvM/7E/PVjU8jhdC3gv0/VuqHouInCbnHK8s2crKbQd48MqedGvRkJk/OQezys7irjveC/TvnAJfe6e7i0js2bKngN+8upyFa3cxILsJhcU+UhLjwx7m4MlAN8qHePj/I4tI9PP5HS98spE/vr2aOIM/fK8XNw7IIi4ucjLGe4Eelwj+ovJjEZHTtOdwERPnrmFghyb891W9ad0oNdwlfYf3At35qh6LiASp2OfntS++5uq+bWialsy/f3Y+bZukRsTySmUU6CIilVi+dT+/emUZX31zkGYNUxjSpSlZGfXCXVaVvBXoNb3sX0SkgsJiH4++u5anF64no34Sf72pH0O6hOay/drmrUAXETlNP34hl4Vrd3Fd/7b8ekR30lOj5zic9wM9PjncFYhIhDtYWExifBwpifHccWEnxg7pyLmdMsNdVo156ybRKY2/+9oDO+q+DhGJGu9/tYNLJy3g8ffWAjCoQ0ZUhjl4LdAnbDxxmmJK47pryiUiUWfP4SLueXkpt05dTP3kBC7u0TzcJZ027y259LkW8ufBL1aFuxIRiVAL1+7k7mlL2X+kmJ9f1Jk7LuxIckLdN9MKNe8FuuPEXYVERCrRLC2F9pn1eeiqXnRr0TDc5YSMt5ZcjlOgi8gJzjmmLdrMA6+tAKBrizT+NfZsT4U5eHGGrmZcIlLG5t0FTJiZx8frdjOoQ2Q10wo17wX6zjVweGfgzva6+bFIzPL5Hc99tIE/vbOahLg4/t9Vvbmuf9uIaqYVat4K9Nyp8PXiwPM37wo8KtRFYtKew0U89t5azu2YyUNX9aJleuQ10wo1b62hr3q96rGIeFpRiZ/pi7fg9zuapiUz++fn88wtOTER5uC1GXr3UbBuXvmxiMSEZVv2ce8reaz+9iAt0lMY3KUpbZtEdjOtUPNWoItIzDlS5GPi3NX87cMNNEtL4ZmbcxgcJc20Qs1bgV7ZkovW0EU87ccv5PJh/i6uH5DFr0d0o2FK9DTTCjVvBbqWXERiwoHCYpJKm2n9bGgnfnphR87pGJ39V0LJWwdFc0ZDq36QkAIjH9PsXMSD3lv1LZdMXMBjpc20BnbIUJiX8tYMHdCFRSLetPvQUf7rjS+ZtWwb3VqkMbxni3CXFHG8Fei5U2Hb54HnOg9dxDMWrNnJ3S8v5WBhMfdc3IWfXNCRpARvLTCEgrcCXQdFRTypRXoKnZo24KGretGleVq4y4lY3voVV/EgqA6KikQlv9/xz882c9+rywHo0jyN6WPPVphXw1szdBGJeht3HWbCzDw+Xb+HsztkHG+mJdXzVqBryUUkavn8jmc/3MAjc1eTGBfH//5Hb67t39aTXRFrS1BLLmY23MxWm1m+mU2o5P10M3vDzJaZ2UozuzX0pQZBSy4iUWvP4SL+PG8t53VqytxxQ7huQJbCvIaqDXQziweeBC4DegDXm1mPCpvdAXzpnOsDXAA8YmZJIa61ejmjoUUfSKyn89BFosDREh8vLdp8opnWXefz9M39aJGeEu7SolIwSy4DgHzn3HoAM5sGjAK+LLONA9Is8Ou0AbAHKAlxrcFpkg2+owpzkQj3xea9jJ+Rx5pvD9G6USqDuzSlTePYaqYVasEEemtgS5nxVmBghW2eAGYB24A04FrnnL/iNzKzMcAYgKysrFOpt3pOFxaJRLKCohIeeWcNz360gRYNU3hudP+YbaYVasEEemWLWBVT81JgKTAU6AjMNbOFzrkD5f6Qc1OAKQA5OTm1lLwO3VNUJHKNeWEJH+bv4j8HZTF+eDfSYriZVqgFE+hbgbZlxm0IzMTLuhX4X+ecA/LNbAPQDVgUkiprSgdSRCLK/iPFJCcEmmn9/KLO/GxoJwZ2yAh3WZ4TzFkui4HOZta+9EDndQSWV8raDFwEYGbNga7A+lAWGrS9G2HvpkAbABEJu7lffsslk+bz6LuBZloD2jdRmNeSamfozrkSM7sTmAPEA88651aa2djS9ycDfwCmmtlyAusd451zu2qx7srlToVvAleWqZeLSHjtOnSUB2et5M287XRrkcaI3mqmVduCurDIOTcbmF3htcllnm8DLgltaadAFxaJRIQPVu/g7peXUnDUxy+GdWHsBR1JjPdWp5FI5K0rRXWDC5GI0KpRKl2bp/HQ93rRWf1X6oy3fmXmjIbmvSCpgS4sEqlDfr/jxU838euZJ5ppvXz72QrzOuatGTpAetvAWS4Kc5E6sX7nISbMWM6ijXs4v3OmmmmFkfcCfd9m2Fd6lotCXaTWlPj8PL1wA5PeXUNKQhwPX3MG1/Rro/4rYeStQM+dCjtWBp7rLBeRWrW3oJjJ89dxYdem/GFUL5o1VP+VcPPWGnplZ7mISMgcLfHxj882HW+m9dZd5/PXm3IU5hHCWzP0eplVj0XklC3ZFGimlb/jEO2a1Oe8zpm0apQa7rKkDG8FesGuqsciUmOHj5bwp3dWM/XjjbRKT+X5Hw7gvM6aLEUibwW6ZugiITfmxVw+yt/NLWe341fDu9Eg2Vux4SXe+slohi4SEvsLiklODDTTuvviLtx9MfTPbhLusqQa3jooqlvQiZy2t1ds5+JJ85n07hogEOQK8+jgrRm6iJyyHQcL+d3rK3lrxTf0aNmQK85oFe6SpIa8FehqziVySt5fvYO7py3lSLGPX13alTGDO6iZVhTyVqCrOZfIKWnTKJWerRry+1G96NSsQbjLkVPkrV/BOaMhozMkN1RzLpEq+P2O5z/eyIQZeQB0bp7GP388SGEe5bw1Qwdo2ArqZSjMRU5i3c5DjH8lj9xNexncpamaaXmI9wL9O/evFhGAYp+fKQvW89h7a0lNjOdP3+/D1X1bq5mWh3gw0NFNokUqsf9IMVMWrOfi7s148MqeNEtT/xWv8WagiwgAhcU+/pW7hRsHtiOzQTJv330+LdPVf8WrvBfoTksuIgCLN+5h/Ct5rN91mPaZDTivc6bC3OO8F+gAaMlFYtehoyX88e2veOGTTbRpnMqLt6mZVqzwaKCLxK4xL+Tyyfrd3HpuNr+8pCv11UwrZugnLeIB+wqKSE6IJzUpnl9c0gUw+rVrHO6ypI5568KiY3SWi8SQ2cu3c/HE+Txa2kyrX7smCvMYpRm6SJTacaCQB15fwZyV39K7dTqjzmwd7pIkzLwX6DrLRWLAvK++5e5pSzla4mfCZd340XntSVAzrZjnvUAHdJaLeF1Wk3r0aduI/7qyJx2aqv+KBOhXukgU8Pkdz364gXtfWQZAp2ZpvHjbQIW5lOPBGbpDM3TxkrXfHmT8jDw+37yPC7uqmZacnAcDHZ3lIp5QVOLnr/PX8ed5+dRPjufRa89k1Jmt1ExLTiqoJRczG25mq80s38wmnGSbC8xsqZmtNLP5oS1TJPYcKCzmbx9t4JKezZk7bgjfO0udEaVq1c7QzSweeBIYBmwFFpvZLOfcl2W2aQT8BRjunNtsZs1qqd7qHdgOR/ZA7lT1RJeoU1js4+XFW7hpUKCZ1py7B9O8oboiSnCCWXIZAOQ759YDmNk0YBTwZZltbgBmOuc2AzjndoS60KDkToW96wPP37wr8KhQlyjx2frdTJi5nA27DtOpWQPO7ZSpMJcaCWbJpTWwpcx4a+lrZXUBGpvZB2a2xMxuruwbmdkYM8s1s9ydO3eeWsVV+eypqsciEehgYTH3v7aca6d8Sonfzz9+NJBzO6mZltRcMDP0yhbtKl69kwD0Ay4CUoFPzOxT59yacn/IuSnAFICcnJzQXwFU8aIiXWQkUWDMC0v4dMNubjuvPb+4pAv1krx5roLUvmD+5mwF2pYZtwG2VbLNLufcYeCwmS0A+gBrqEuDfnpiqeXYWCQC7TlcRGpioJnWLy/tihn0zVL/FTk9wSy5LAY6m1l7M0sCrgNmVdjmdeB8M0sws3rAQGBVaEsNQs5oaNweUhrByMe0fi4RxznHrGXbuHjifCYdb6bVWGEuIVHtDN05V2JmdwJzgHjgWefcSjMbW/r+ZOfcKjN7G8gD/MAzzrkVtVl41XRql0Seb/YXcv9rK3h31bf0aZPOf/RVMy0JraAW65xzs4HZFV6bXGH8MPBw6Eo7BblTYe+GwHOd5SIR5L1VgWZaxX4/943ozg/Pa098nCYeElre6uWy6vWqxyJh0i6jPn3bNebtuwbz48EdFOZSK7wV6N1HVT0WqSM+v+OZhev5xfRjzbQa8PwPB5CdWT/MlYmXeSvQc0ZD42xIaayDohI2a749yNVPfcxD/17F3oIiCot94S5JYoT3Tnit3ywQ6gpzqWNFJX6e+mAdT7y/lrSURB677kyu7KNmWlJ3vBfogM5ykXA4UFjM1I83MKJ3S347sgcZDZLDXZLEGI8GukjdOFLk46VFm7nlnOzjzbSaqf+KhIkHA12X+0vd+HjdLibMWM7mPQV0bZHGuZ0yFeYSVh4MdHSDC6lVBwqL+Z/ZX/HSos20y6jHSz8exNkdM8JdlogHA10NuaSWjXkhl0Ub9nD74A7cfXEXUpN0OziJDN4LdEAHRSXUdh86Sr2kBFKT4rl3eDfizejTtlG4yxIpx1vnoYuEmHOO15d+Xa6ZVt+sxgpziUgenKFryUVCY/v+I9z/6gre+2oHZ7ZtxDX92oS7JJEqeTDQ0UFROW1zv/yWe15eis/veGBkD0afk63+KxLxvBnoIqepfWZ9crIb8/sre5GVUS/c5YgExXtr6DrLRU5Bic/PlAXrGPfyUiDQTGvqrQMU5hJVPDpD1/8aS/BWbT/A+Bl55G3dz7AezSks9pGSqFMRJfp4NNBFqne0xMeT76/jL+/n06heIk/e0JcRvVuomZZELQ8GupZcJDiHCkv4+6ebuLJPKx4Y2YPG9ZPCXZLIafHeGvrhXbBtSeB2dCIVFBSV8MzC9fj8jozSZloTrz1TYS6e4K0Zeu5U2L8l8Fz3FJUKPsrfxYSZeWzZc4QeLRtyTqdMmqapxa14h7dm6LqnqFRi/5Fixr+Sx43PfEZCXBwvjxnEOZ0yw12WSMh5a4befRSsm1d+LDHv9hdzWbxxL2OHdOTuizvrDBbxLG8Fes5oWPBHKCmEob/VcksM23nwKPWT46mXlMD44d1IiIujd5v0cJclUqu8teQCUD8T2vRXmMco5xwzP9/KsEnzmTQ30EzrrKzGCnOJCd6aoUtM+3rfEe57dTkfrN5J36xGXNu/bbhLEqlT3gt0Xfofk95Z+Q33vLwUBzx4RQ9uOlvNtCT2eC/QAV36Hzucc5gZHZs1YFCHDB68sidtm6j/isQm762hS0wo8fl56oN13FPaTKtj0wb8bXR/hbnENA/O0LXk4nVfbjvAvTOWseLrA1zaU820RI7xYKCjG1x4VGGxjyfm5TN5/joa1UviqRv7clnvluEuSyRieC/QD++GA9sDbQB06qKnHD5awj8XbWbUma15YGR3GtVT/xWRsoJaQzez4Wa22szyzWxCFdv1NzOfmV0TuhJrIHcqHNwGBbsCvVzUoCvqHT5awpQF644305p7z2Ae+UEfhblIJaoNdDOLB54ELgN6ANebWY+TbPd/wJxQFxk09XLxlAVrdnLJpAX8z1tf8dmG3QBkNFAzLZGTCWaGPgDId86td84VAdOAypqk/AyYAewIYX01U7F3i3q5RKV9BUX88l/LuPnZRSQnxvGv28/mnI5qpiVSnWACvTWwpcx4a+lrx5lZa+AqYHJV38jMxphZrpnl7ty5s6a1Vi9nNKS1gnqZMPIxraFHqTEvLuHVL77mjgs7Mvvn55OT3STcJYlEhWAOilZ2ykjFcwMfBcY753xV3b7LOTcFmAKQk5NTO+cX1msCjbMV5lFmx8FCGiQnUC8pgd+M6E5ivNGzlfqviNREMIG+FSjbFKMNsK3CNjnAtNIwzwRGmFmJc+61UBQp3uWc45UlW3no36v4fr823D+yB2e2bRTuskSiUjCBvhjobGbtga+B64Abym7gnGt/7LmZTQXeDFuYq5dL1Niyp4DfvLqchWt30T+7MdcPzAp3SSJRrdpAd86VmNmdBM5eiQeedc6tNLOxpe9XuW4uUpm3V3zDuOlLMeD3o3rynwPbEadmWiKnJagLi5xzs4HZFV6rNMidc6NPv6zToRl6JDvWTKtL8wac2ymT313RgzaN1X9FJBS82ZxLl/5HnGKfnyffz+euaUsB6NC0AU/fnKMwFwkhbwa6RJQVX+9n1BMf8fCc1fic42iJL9wliXiS93q5FOyBwzvVyyUCFBb7eOy9tUxZsJ4m9ZP46039uLRni3CXJeJZ3gr03Klw6JvA8zfvCjwq1MOmoMjH9MVbuLpva+4b0YP0eonhLknE07y15PLZU1WPpdYdOlrC5PmBZlpN6icxd9wQ/nhNH4W5SB3w1gy94jnoOie9Tn2wegf3vbqCbfuP0KdNI87umEGT+uqKKFJXvDVDH/TTqsdSK/YeLmLc9KWMfm4xqUnxvDL2HM7umBHuskRijrdm6Js++u5Ya+i17va/L+HzTXv5+dBO3DG0E8kJuh2cSDh4K9Dz51Y9lpDZcaCQ+skJ1E9O4L4R3UmMj6NHq4bhLkskpnlryaVxh6rHctqcc0xfvIWLJs5n4tw1APRp20hhLhIBvDVDT02veiynZfPuQDOtD/N3MaB9E25UMy2RiOKtQO8+CtbNKz+WkHh7xXbueXkZ8XHGQ9/rxQ0DstRMSyTCeGvJJWc0xCdDfBL0/oEOiIaAKz31s2uLhgzp0pR37hnMfw5SZ0SRSOStQM+dCr6j4CuC5dMDYzklRSV+/vzeWn4+bSnOOdpn1mfyTf1o1Sg13KWJyEl4K9B1pWhI5G3dx5VPfMgjpQc9i3z+MFckIsHw1hq6rhQ9LYXFPibNXcPTC9fTNC2Zp2/OYViP5uEuS0SC5K0Zuq4UPS0FRT5eWbKVa/u35Z17hijMRaKMt2boUmMHC4t58dNN3D64I03qJ/HuuCE0Vv8VkajkrRn6qterHks58776lksmLeBPc1azaMMeAIW5SBTzVqBXPO9c56FXaveho9w17Qt+ODWXtJQEZvxEzbREvMBbSy5qzhWUn/z9c77Yspe7L+7MTy/oRFKCt36vi8QqbwW6mnOd1Df7C0lLCTTTemBkD5IS4ujaIi3cZYlICHlratZpWNXjGOSc46VFmxlWpplW7zbpCnMRD/JWoF/9dODR4qBVvxPjGLVp92FuePozfj1zOb1ap3Pz2e3CXZKI1CJvLbkcu9Tf+WHbksA4RtfQZy/fzrjpS0mMi+N//qM31/Vvi5n6r4h4mbdm6Dpt8Xgzre4tGzK0WzPeGTeY6wdkKcxFYoC3Aj2GT1ssKvHz6LtruPOlL4430/rLjf1oma5mWiKxwluBfmx5pVE2jHwsZpZblm7ZxxV//pBH311LQpypmZZIjPLWGjoABmd8PybC/EiRj4lzV/O3DzfQLC2Fv92Sw0Xd1X9FJFZ5MNAdEBvrxYXFPl79YhvXD8hiwmXdSEtJDHdJIhJGQS25mNlwM1ttZvlmNqGS9280s7zSr4/NrE/oS60BDx8APFBYzBPz1lLi89O4fhLvjRvCf1/VW2EuItXP0M0sHngSGAZsBRab2Szn3JdlNtsADHHO7TWzy4ApwMDaKLhKx/ufezPQ3/3yW+57bTk7Dx6lX7smnN0xg/R6CnIRCQhmyWUAkO+cWw9gZtOAUcDxQHfOfVxm+0+BNqEsMmjHAj3vZUhr6Zl19N2HjvLgG1/yxrJtdGuRxtM353BGm0bhLktEIkwwgd4a2FJmvJWqZ9+3AW9V9oaZjQHGAGRlZQVZYg0smRp43LsB3rwr8NwDoX6smda4YV0YO6SjmmmJSKWCCfTK1i8qvbebmV1IINDPq+x959wUAssx5OTkhP7+cKtmVRi/HrWBvn3/ERqmJFI/OYHfXhFoptWlufqviMjJBTPV2wq0LTNuA2yruJGZnQE8A4xyzu0OTXk11P2KCuPou7DI73f847NNDJu4gEfeCTTT6tU6XWEuItUKZoa+GOhsZu2Br4HrgBvKbmBmWcBM4Cbn3JqQVxkjNuw6zIQZeXy2YQ/ndspg9DnZ4S5JRKJItYHunCsxszuBOUA88KxzbqWZjS19fzLwWyAD+Etpz5AS51xO7ZV9ElG85PLvvEAzraSEOP549Rl8P6eN+q+ISI0EdWGRc242MLvCa5PLPP8R8KPQlnYKuo2E9R+cGEfBkotzDjOjZ6uGDOvRnAdG9qB5w5RwlyUiUchbp0ucdVPgsUmHiO/lcrTEx8R3VnPHPz/HOUd2Zn2euKGvwlxETpm3Av3YyTdn3RTRYf755r2MfPxDHp+XT0pCvJppiUhIeLCXC/DF3yG1ScSFekFRCX+as4bnPt5Ay4YpPHdrfy7s2izcZYmIR3gr0D9/MfC4Z11EXlh0tNjPG3nbuGlQO+4d3o0Gyd76zy8i4eWtJZev3iw/joA7Fu0/Uszj751opvXuuCH8flQvhbmIhJy3UiU1o/y4XmZ46ig1Z+U3PPDaCnYfLmJg+yYM7JBBeqqaaYlI7fBWoB+pcIFqwa6wlLHz4FEenLWSfy/fTveWDfnbLf3p3SY9LLWISOzwVqB3vQw2zD8xDtN56D/9xxKWbdnPLy/pwu1DOpIY762VLRGJTN4K9Mp7htWJr/cdIT01kQbJCfzuip4kJ8TRWf1XRKQOeWvq+FWFrr11cFDU73e88MlGLpk4n4llmmkpzEWkrnlrhl6vSYVx7R4UXbfzEBNm5LF4417O75zJredm1+rniYhUxVuBXlB3B0XfzNvGuOnLSEmI4+FrzuCafmqmJSLh5a1Ar4PTFo810+rdOp3hPVtw/8juNEtT/xURCT9vraEfqTAjD+EMvbDYx8NzvuInfw8002qXUZ/Hrz9LYS4iEcNbgZ7auPw4RDP0JZv2cPnjC3ny/XXUT05QMy0RiUjeWnL59svy42/yTuvbHT5awsNzVvP8JxtplZ7K8z8cwJAuTU/re4qI1BZvBXrh/vLjI/tO69sV+/zMXr6dmwe141dqpiUiEU4JVcG+giKe+2gjPxvaiUb1knj3F0NomKL+KyIS+bwV6MkNyx8ITalZ/5S3lm/ngddXsregiHM6ZjCwQ4bCXESihrcOip55ffnxoJ8G9cd2HChk7ItL+Mk/Pqd5w2Rm3XkuAztkVP8HRUQiiLcCvedVgcfE+tD7B0Hf3OKOf37OvNU7GD+8G6/fcS49W6kzoohEH28tuayYEXgsPgzLp0O7c08a6lv3FtCoXhINkhN48MqepCTG07Fpg7qrVUQkxLw1Q1/6UvnxZ099ZxO/3zH1ow1cMmkBj7yzGoCerdIV5iIS9bw1Q6/YPteVH+fvCDTTyt20lyFdmnLbee3rsDYRkdrlrUBPSC0/Tjox6561bBu/nL6MesnxTPxBH646q7WaaYmIp3gr0PdtLD/e+RV+vyMuzujTJp0RvVtw3+U9aJqWHJbyRERqk7fW0MsssTjgiM8x9u9LjjfTevS6sxTmIuJZ3gn0LYvAd7TcSzuK69G4XhLFvvDdmk5EpK54Z8nlzXE4oOyqeP2ss/i/a84IV0UiInXKOzP0XWuPPz02H88cfm94ahERCYOoD/S9h4uY+M5q/CkNj8/ODbCUxtB2QDhLExGpU0EFupkNN7PVZpZvZhMqed/M7PHS9/PMrG/oSy3POce/87YzbNJ8/vLBOnY3O6f8Bp2H1XYJIiIRpdo1dDOLB54EhgFbgcVmNss5V/ZuEpcBnUu/BgJPlT6G3pZFHFz9Pk+ub8Hk9Zm8nzqedklbiNtQYbvVb9XKx4uIRKpgDooOAPKdc+sBzGwaMAooG+ijgBeccw741MwamVlL59z2kFa7ZRFMvZwGviLudfDLFIh35Q+EHld0MKQfLSIS6YJZcmkNbCkz3lr6Wk23wczGmFmumeXu3LmzprXCxoXgLwmskVvgt5Gu9RQRCQgm0CvLzIondgezDc65Kc65HOdcTtOmp3BvzuzzIT4ZLB5LSIX0rJp/DxERjwpmyWUr0LbMuA2w7RS2OX1tB8AtswIz9ezzA+MnBsCu1d/d9sH9331NRMTDggn0xUBnM2sPfA1cB9xQYZtZwJ2l6+sDgf0hXz8/pu2A8qcj3rmoVj5GRCTaVBvozrkSM7sTmAPEA88651aa2djS9ycDs4ERQD5QANxaeyWLiEhlgrr03zk3m0Bol31tcpnnDrgjtKWJiEhNRP2VoiIiEqBAFxHxCAW6iIhHKNBFRDzCnAvPzR/MbCew6RT/eCawK4TlRAPtc2zQPseG09nnds65Sq/MDFugnw4zy3XO5YS7jrqkfY4N2ufYUFv7rCUXERGPUKCLiHhEtAb6lHAXEAba59igfY4NtbLPUbmGLiIi3xWtM3QREalAgS4i4hERHeiReHPq2hbEPt9Yuq95ZvaxmfUJR52hVN0+l9muv5n5zOyauqyvNgSzz2Z2gZktNbOVZja/rmsMtSD+bqeb2Rtmtqx0n6O6a6uZPWtmO8xsxUneD31+Oeci8otAq951QAcgCVgG9KiwzQjgLQJ3TBoEfBbuuutgn88BGpc+vywW9rnMdvMIdP28Jtx118HPuRGB+/ZmlY6bhbvuOtjn3wD/V/q8KbAHSAp37aexz4OBvsCKk7wf8vyK5Bn68ZtTO+eKgGM3py7r+M2pnXOfAo3MrGVdFxpC1e6zc+5j59ze0uGnBO4OFc2C+TkD/AyYAeyoy+JqSTD7fAMw0zm3GcA5F+37Hcw+OyDNzAxoQCDQS+q2zNBxzi0gsA8nE/L8iuRAD9nNqaNITffnNgK/4aNZtftsZq2Bq4DJeEMwP+cuQGMz+8DMlpjZzXVWXe0IZp+fALoTuH3lcuAu55y/bsoLi5DnV1A3uAiTkN2cOooEvT9mdiGBQD+vViuqfcHs86PAeOecLzB5i3rB7HMC0A+4CEgFPjGzT51za2q7uFoSzD5fCiwFhgIdgblmttA5d6CWawuXkOdXJAd65Nycuu4EtT9mdgbwDHCZc253HdVWW4LZ5xxgWmmYZwIjzKzEOfdanVQYesH+3d7lnDsMHDazBUAfIFoDPZh9vhX4XxdYYM43sw1AN8CrNw4OeX5F8pLL8ZtTm1kSgZtTz6qwzSzg5tKjxYOozZtT141q99nMsoCZwE1RPFsrq9p9ds61d85lO+eygVeAn0ZxmENwf7dfB843swQzq0fg5uur6rjOUApmnzcT+D8SzKw50BVYX6dV1q2Q51fEztBdDN6cOsh9/i2QAfyldMZa4qK4U12Q++wpweyzc26Vmb0N5AF+4BnnXKWnv0WDIH/OfwCmmtlyAssR451zUdtW18xeAi4AMs1sK/A7IBFqL7906b+IiEdE8pKLiIjUgAJdRMQjFOgiIh6hQBcR8QgFuoiIRyjQRUQ8QoEuIuIR/x+1lszTBxWrEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nb_probs =nb_model.predict_proba(X_test)\n",
    "\n",
    "# keep probabilities for the positive outcome only\n",
    "probs = nb_probs[:, 1]\n",
    "# calculate AUC\n",
    "auc = roc_auc_score(y_test, probs)\n",
    "print('AUC: %.3f' % auc)\n",
    "# calculate roc curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, probs)\n",
    "# plot no skill\n",
    "pyplot.plot([0, 1], [0, 1], linestyle='--')\n",
    "# plot the roc curve for the model\n",
    "pyplot.plot(fpr, tpr, marker='.')\n",
    "# show the plot\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "e6b1ef2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD4CAYAAAAn3bdmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAyXklEQVR4nO3dd3gVRRfA4d+5SQgJTUINAelFugaxRGkBBBUIHURBUUK3IR8dsaCogIqF3i2AdCmioBSRjki1gLQAgiIlhJA63x93E28gFRLCXc7rs4/3np3dnUU8mczOzogxBqWUUu7Bkd0VUEoplX6atJVSyo1o0lZKKTeiSVsppdyIJm2llHIjnll9gYY5X9PhKeoaKy8Mye4qqFuQl7en3Og56snwdOecteb1G77ezaYtbaWUciNZ3tJWSqmbScTtGs8ZoklbKWUr4qFJWyml3IbNG9qatJVSNmPzrK1JWyllKzbP2Zq0lVL2Ig57Z21N2kope7F5U1uTtlLKVhza0lZKKTdi75ytSVspZS/ap62UUm7E5l3amrSVUjZj86ytSVspZSsOfY1dKaXciLa0lVLKfdg8Z2vSVkrZi07NqpRS7sTeOVuTtlLKXnSctlJKuRFN2kop5Ua0T1sppdyJzZcr16StlLIVu7e0bf4zSSl1uxFJ/5b2uWSaiJwRkb0usbkissvajojILiteSkQiXfZNcDkmUET2iMhBERkn1k8WEfG2zndQRLaISKm06qQtbaWUrWTyg8gZwMfArISAMaZ94rVExgAXXMofMsbUTOY844FQYDOwAmgCrASeBc4ZY8qJSAfgHaB9Mscn0pa2UspWxCHp3tJijFkP/JvsdZyt5XbAl6nWR8QfyGuM2WSMMTh/AIRYu1sAM63P84FgSaN/R5O2UspeMtA/IiKhIrLdZQvNwJUeBk4bY/5wiZUWkZ9FZJ2IPGzFAoAwlzJhVixh33EAY0wszlZ7gdQuqt0jSilbychzSGPMJGDSdV6qI0lb2aeAO40xZ0UkEFgsIlVI/h1NY/07tX3J0qStlLKVmzF6REQ8gVZAYELMGBMFRFmfd4jIIaACzpZ1cZfDiwMnrc9hQAkgzDpnPlLojkmg3SNKKXtxZGC7fg2BX40xid0eIlJIRDysz2WA8sCfxphTQLiI3G/1V3cGlliHLQW6WJ/bAN9b/d4p0pa2UspWHI7Ma4uKyJdAPaCgiIQBrxpjpgIduPYBZB3gdRGJBeKAHsaYhFZzT5wjUXxwjhpZacWnArNF5CDOFnaHtOqkSVspZSuSif0HxpiOKcSfTia2AFiQQvntQNVk4leAthmpkyZtpZS92PyNSE3aSilbsXnO1qStlLIXnZpVKaXcic2b2pq0lVK24vDQpK2UUu5DW9q3n7x+Pry7sjMAfkVyEx8Xz/l/LgPQ56HJxMbE3/A1xnzbhZy5ctA7aDIAFe7xp/uoxvRrPDONI1V2qV6zGuXLl0/8Pu6DjwgICEi27L331WLblu03dL0hQwezfft2cufJjUMcDBkylJo1at7QOW8HNs/ZmrSTc/HfSHrcNxGAzkPrEnkpmq8+2JS43+EhxMel+tJSuuQvnIt7G5dj27cHb/hcKut5e3uz4KuFN/Wa/V7uR+PGj7Dxp4289vprLFqw6KZe3x3pg0gFQP/JLQj/N5JyNYvyx89/EXkpKkkyn7yjJ0NbfcHpoxcI7liNlr3uwzOHB79uO8G455cTH39tkp839ic6DXr4mqTtcAjPvdmQGnVK4uXtyZKJ21g+ZQci0PeDR6n+cEn+OnIecQjfzPyZDYsO3JQ/A5XU5csR9H2+LxcvXiQ2Npa+fZ+nQf0GScr8/fffvNK/H5ciLhEXG8ewocMJDAxk408b+fTTT4iOjqZEiRK8+cab+PrmSvFatQJrcfz4MQBmzprBokXO5N26VWueeqozly9fpl//fpw+/RfxcfF0796Dpk2aZt3N38rsnbM1aWdE8fIF+F/T2cTHGzoPrZtsmTsrFqRemyq8UH8acbHxPP/howR3rMZ3n+++puz+LccJalGJGnVLERkelRhv+szdRFy8Qu+HpuCVw4MPfujKjtWHKH+3P0VK3kG3wPHcUTgX03b15puZP2fZ/aqkoqKiaN22FQABAcUZO3osH34wjty5c3Pu3DmeeLIj9evVTzJh0fIVy3nwwSC6h3YnLi6OK1eucO7cOSZNmsjkSVPw9fVl6rQpzJw1k549eqV47bXr1lK+XHn27d/H4sWL+eLzLwFDx04dqVXrXsLCjlO4UCHGfzIegPDw8Cz9s7iVOTzsPaWSJu0MWLdwf7ItZld31y9N+buL8cnGbgB4+3hy/u+IFMt/Pmo9nQY+zJQhqxNjgcFlKVOtCA+3rAxArnzeBJTzo+qDd7J+4X6MgXOnI9i17siN35RKt6u7R2JiYvhw3Ads37EDh0M4c+YMZ8/+Q8GChRLLVK1SlWGvDiU2NpbgBg2oVOkutm3fxqE/D/FUlycTz1Mjhb7qMWPHMHHyRPLn9+P1195g85bNBAcH4+vrC0DD4Ibs2LmDh4IeYvSY0Yx9fwx169QjMDAw2fPdDrRPWyW6EhGd+DkuNj5J31mOnM4/ShHhu89/YeqwNek65661R3j61frcVfu/mRtF4OOXVrJ99aEkZe9rUuFGqq8y2fIVy/j33DnmzZmHl5cXjZs0IioqOkmZWrVqMXP6LNavX8egwYN4+ulnyJc3Lw/c/wDvvTs6zWsk9Gkn2Lx5U7LlSpUqxby581i/YQMfjHufBx94MNWWu63ZPGvb+/eILPTX0fOUr+kPQLmaRSla6g4Adv7wJw+3vIs7CjlbQnny56TwnflSPdcXozbQvl9Q4vftqw/RLLQWHp7O/zwB5fzI6evF3p+O8XDIXYjAHYVzUaNOqcy/MZVu4eGXKODnh5eXF1u3buHkyZPXlDl58iR+fn60adOWVi1bceDAfqpXr8HPu37m2LGjAERGRnLkyJF0XTMwsBZrvv+eyMhILl++zJo1awi8J5AzZ86QM6cPzR5vxtNdnuHAgdv3OUdmLjd2K9KW9nXasOgAjTrVYMKW7vy+4yRhf5wF4Niv/zBjxA+MWvYUDocQGxPHRy+u4MyxCymea+uqg0m6UFZM20mRkncwYXMoiHDhnwhebTuXDYv2c3f90kzZ2YuwP87y67YwIi5GpXhelbUef+xx+vTtTbsO7ahUsRKlS5e5psy2bVuZPmM6nl6e+Pr48tbIt/Hz82PkGyPpP6A/0dExADzfpy+lSpVK85qVK1cmpEULOj7hnMGzdavW3HXXXWzc+COjx47B4RA8PT0ZNnR4pt6rO7F5QxtJY77tG9Yw52tZe4HbTM5cXlyJiCGvnw8f//gcL9SfxrnTKfeZ36pWXhiS3VVQtyAvb88bTrnPNZqW7pwz5buubpfitaXtZkYueoJc+XLilcODz95e75YJW6mspK+xq1uKvjGpVBps3j+iSVspZSs2z9k6eiSjXpnYnK+OvcLkHT0TY2WqFWHc2q5M3t6DNxZ0wDdPjiTHFC6Rl6//GUTbFx9IjJW/25/J23swc19feo9pkhiv9tCdjN8UyqpLw3i45V1Zf0Mqy128eJGXXn6RZs0fp1mLZuz6ZRf9+vejddtWtG7bisZNGiW+tBMTE8PgIYNo2SqEZi2aMXnK5GyuvfvJzNEjIjJNRM6IyF6X2AgROSEiu6ztUZd9g0TkoIj8JiKPuMQDRWSPtW+ctcAvIuItInOt+BYRKZVWnTRpZ9Cq2bsY1PyzJLF+45sxZdgautWawMalv9Lu5aAk+3u++whbV/2RJPbCuMcY23sZXap8REA5P+5tXA6AM8cv8G63xXw/d0/W3oi6aUa98zZBQQ/x9dJlLJy/gDKlyzDmvTEs+GohC75aSKOGjWgY3BCAb79dRXRMDIsWLmbenHl8NX8eJ06cyOY7cDMi6d/SNgNokkz8fWNMTWtb4bysVMa5MG8V65hPE1ZnB8YDoThXaC/vcs5ngXPGmHLA+8A7aVUozaQtIpVEZID10+FD6/Nt2wTc8+Mxws9FJokVr1CQ3RucY253rPmTh0P+++N5sFlFTh0+z9EDfyfG/IrmxjevNwe2hAHw3ee7CWpeCYDTRy9weO+ZNN+8VO7h0qVL7Nixg9atWgPg5ZWDvHnzJu43xvDNqlU82vQxwPlyVuTly8TGxhIVFYWXlxe5c6c8J4m6VmbmbGPMepyrpKdHC2COMSbKGHMYOAjUFhF/IK8xZpNxDtebBYS4HJPwoGo+EJzQCk9JqklbRAYAc3BOwbIV2GZ9/lJEBqbzRmzvyL4zPPh4RQDqtKpMoeLO/ylz+nrRoV8Qs0auTVK+YLE8/HPiYuL3v09cpGCxPDetvurmCQs7Tn6//AwdNoQ27Voz/NXhXL58OXH/jh07KFCgACVLlgSgUaPG+Pj6Uj+4Ho0aN+TpLk+TL98d2VR79yQejvRvIqEist1lC03nZfqIyG6r+yS/FQsAjruUCbNiAdbnq+NJjjHGxAIXgAKpXTitlvazwL3GmFHGmM+sbRRQ29qXLNc/iBNxNzansDsY3X0JzXvcy6c/dcM3jzex0XEAdB5WjwUfbeZKREyS8sn9IM3q8fIqe8TGxXHgwAHat+vA/HkL8PHxYeq0KYn7V6xcwaNNE7tE2bN3Dx4OB9+v/oFvVq5i5syZHA87ntypVQoy0tI2xkwyxtRy2Sal4xLjgbJATeAUMCbh0smUNanEUzsmRWmNHokHigFHr4r7W/uSZd34JLg9Xq45/vtZBj7u7OcOKOfHfU2cE+XfVTuAOq0q0+2tRuTOl5P4eEP0lVg2LD5AwYD/fkUuFJCXs6cuZUvdVdYqWqQIRYoUoXr16gA0btSYKVbSjo2NZfWa1cybMy+x/IoVywkKeggvLy8KFChAzbvvZt++fZQoXiJb6u+Osvr1dGPM6cRriUwGlllfwwDX/1DFgZNWvHgycddjwkTEE8hHGt0xaSXtF4E1IvIH/zX77wTKAX3SOPa2cUchX87/fRkReHJQHZZNcf528VLwjMQyCYspLJmwDYDI8Cjuqh3Aga0naNSpOos/3ZodVVdZrGDBQhQtUpTDhw9TunRpNm/ZTNkyZQHn5E9lSpemaNGiieX9/f3ZunULzR5vRmRkJLt3/8JTTz6VXdV3S2l0CWfG+f2NMaesry2BhJElS4EvRGQszsZueWCrMSZORMJF5H5gC9AZ+MjlmC7AJqAN8L1J49fuVJO2MeYbEamAszskAGdTPgzYZoyJy9it2sPgWa2o8XAp8hX05cuDLzHzzbX45MpBix73AvDj4gN8M3NXmuf58Pnl9J8cgrePJ1tXHWTrKudCCBUDizFibnty58/JA49WoMuwejx3z/isvCWVxQYPGsyAQQOIiYmhRPHivPHGmwCs/GYlTV26RgA6dujI0GFDCWnVAmMMIS1aUrFCxeyotvvKxJwtIl8C9YCCIhIGvArUE5GaOLsxjgDdAYwx+0RkHrAfiAV6u+TJnjhHovgAK60NYCowW0QO4mxhd0izTjr3iMoOOveISk5mzD3yQsc56c45H37Zwe1exdE3IpVS9uKmU66mlyZtpZSt6GvsKpHDIUzYHMqbCzsCULZ6ET5a9ywTtnTnk43dqFir2DXHeHl78PGG55i4tTtTdvak87B6ifvqtKrMlJ09+fbycCrc458Yr/JACSZt68EnPz5HsTLOIaC58nkz6utOWXuD6roNHT6UOnUfJqRlixTLbN22ldZtW9GiZXOefqZLYnz2Z7MJadmCFi2bM3v2rMT42PfH0LJ1SwYNHpQYW/r1UmZ/NjtrbsImRCTdmzvSpJ0BLfvcx7Hf/kn83u2tRswauY4e901k5us/EPpWo2uOiYmK45UmM+leeyLda0/k3kZluau2c1z9kX1nGNF+Hnt+TDqiss0LD/Bah3lMffV7moU6H3A+OaguX7z7YxbenboRIc1DmDB+Yor7L168yJsj3+DjcR+zZNFSxoweC8Aff/zBggXz+fKLOSz4aiHr1q/j6NGjhIeHs2vXLhYtWER8fBy///47V65cYcmSxXRon+azqtubQ9K/uSFN2ulUMCAP9zUtz4rpO/8LGkOuvN4A5MqXk7Onkl8BO+HlGk8vB55eHiQ8+z322z+JK964iouJI4ePJzl9vIiLicO/TH4KFsuT+Kq8uvXUqlWLfPlSXlZuxYrlNAxuiL+/87exAgWcL739efhPqlevgY+PD56entSqVYs1a1bjcDiIiYnBGMOVqCg8vTyZPmManTo9iZeX1025J3eVuVOP3Hq0Tzuder3XhMmDVyeZwe/TV1YxatmThI5qhEOE5+tPS/ZYh0P4dFMoAWX9WDJhG79uS30CoC/f+5GXP2lG1JUYRnVdRPe3GzPjtR8y9X7UzXXk6BFiY2N5uuvTXI6IoFOnJ2nRvAXlypVj3Ecfcv78eby9vdmwYQNVqlQhV65cNGrYiDbtWnP/ffeTJ3ce9u7de/su1psB4mHvtqgm7XS4r2l5zv8dwR8/n6JGnZKJ8WahtRjffxUbFh+gbuvKvDKhOf979Nr+xvh4Q4/7JpIrnzevzWtPqcqFOLL/72vKJTi0+zR9604FnFO1nj0VDgJDZ7cmNjaeCQO+5fwZXbHGncTFxbF//36mTJ5KVFQUnZ56ghrVa1C2TFm6PvMs3UKfw9fXlwoVK+Lh4ZwYrmvXZ+na1TlbxPBXh9Ond1/mL5jPpk0/UaFCBbqH9sjOW7pluWtfdXrZ+0dSJqn64J088FhFPvvtBYbMakPNeqUZOL0ljZ+swYbFzlWv1y3YT8VaAameJ+JCFL+sP5o4DWt6dBpYh8/eXk/noXWZ+cZa1nyxm5a977uR21HZoEiRIgQFPYSvry/58+cnMLAWv/3+G+BcnPerefOZOWMW+fLmo+SdJZMcm7CyesmSJfn6a2d/+B8HD3L0qHaXJUcc6d/ckZtW++aaOmwNHcu9z5MVP2Rk5/nsWnuYUc8s4p9T4Ykt77vrl+bEwWv7p/MV9CVXPme/d46cntzToHSSh5mpafxUDbas/INL56/g7eNFfLwh3hhy+mifprupX78BO3fuIDY2lsjISPbs3k0Za/X2s2edf29OnTrJmjWrafpo0rckP/rkI/r07kNsbCxx8c4X7BwiRF5JOkWwcrL76BHtHrkB7/f6ml6jm+Dh6SD6Sizv93bOG1PAPzcvj2/OkJAv8CuamwFTQnB4OBCHsG7BPrasdC6IENS8En3GNiVfIV9GLnqCQ7v/YmCzzwHw9vGk8ZM1GPCYcyKq+eM2M2JOO2Ki4xjZeUH23LBKUf//vcK27ds4f/48wQ0b0KtXb2JjYwFo3649ZcuUJSjoIVq1aYlDHLRu1Zry5Z0Ti7308oucv3AeT09PhgweSr68/z3QXPP9GqpWqUrhwoUBqFG9Ji1bhVChQgUqVax082/UHbhpMk4vfY1dZQt9jV0lJzNeYx/Qa3G6c847n4a4XYbXlrZSylZ09IhSSrkRd+2rTi9N2kopW7F5ztakrZSyGTd9PT29NGkrpWxFu0eUUsqNiIcmbaWUchva0lZKKTeS1auxZzd7D2hUSt1+JANbWqcSmSYiZ0Rkr0vsPRH5VUR2i8giEbnDipcSkUgR2WVtE1yOCRSRPSJyUETGifXrgIh4i8hcK75FREqlVSdN2kopW8nkuUdmAE2uin0HVDXGVAd+Bwa57DtkjKlpba7TMI4HQoHy1pZwzmeBc8aYcsD7wDtpVUiTtlLKVsQh6d7SYoxZD/x7VexbY0ys9XUzUDzV+oj4A3mNMZuMc96QWUCItbsFMNP6PB8IljR+mmjSVkrZSkaStoiEish2ly00g5frCqx0+V5aRH4WkXUi8rAVCwDCXMqEWbGEfccBrB8EF4ACqV1QH0QqpWwlI6NHjDGTgEnXeZ0hQCzwuRU6BdxpjDkrIoHAYhGpQvK95wmTWqW2L1matJVStnIzRvyJSBfgcSDY6vLAGBMFRFmfd4jIIaACzpa1axdKceCk9TkMKAGEiYgnkI+rumOupt0jSilbyeqFfUWkCTAAaG6MuewSLyQiHtbnMjgfOP5pjDkFhIvI/VZ/dWdgiXXYUqCL9bkN8L1JY75sbWkrpWwlM1+uEZEvgXpAQREJA17FOVrEG/jOutZma6RIHeB1EYkF4oAexpiEVnNPnCNRfHD2gSf0g08FZovIQZwt7A5p1UmTtlLKVhyZ+HKNMaZjMuGpKZRdACS7rJQxZjtQNZn4FaBtRuqkSVspZSs2f4tdk7ZSyl507hGllHIjNs/ZmrSVUvYi6ZlUxI1p0lZK2Yq2tJVSyo1k5uiRW5EmbaWUrWhLWyml3InNs7YmbaWUrdg8Z2vSVkrZi47TVkopN2LznK1JWyllLzp6RCml3Ii9U7YmbaWUzWiftlJKuRGb52xN2kope9GWtlJKuRF9EKmUUm7E5g1tXdhXKWUvmbmwr4hME5EzIrLXJeYnIt+JyB/Wv/O77BskIgdF5DcRecQlHigie6x946wFfhERbxGZa8W3iEiptOqkSVspZSsiku4tHWYATa6KDQTWGGPKA2us74hIZZwL81axjvk0YXV2YDwQinOF9vIu53wWOGeMKQe8D7yTVoU0aSulbCUzW9rGmPU4V0l31QKYaX2eCYS4xOcYY6KMMYeBg0BtEfEH8hpjNhljDDDrqmMSzjUfCJY0fppkeZ/2qkvDsvoSyg0Fe43I7iqoW9Ba8/oNn+MmjB4pYow5BWCMOSUiha14ALDZpVyYFYuxPl8dTzjmuHWuWBG5ABQA/knp4vogUillK5KB0SMiEoqz2yLBJGPMpOu9dDIxk0o8tWNSpElbKWUrGWlpWwk6o0n6tIj4W61sf+CMFQ8DSriUKw6ctOLFk4m7HhMmIp5APq7tjklC+7SVUraSmX3aKVgKdLE+dwGWuMQ7WCNCSuN84LjV6koJF5H7rf7qzlcdk3CuNsD3Vr93irSlrZSylczs0xaRL4F6QEERCQNeBUYB80TkWeAY0BbAGLNPROYB+4FYoLcxJs46VU+cI1F8gJXWBjAVmC0iB3G2sDukVSdN2kopW8nM55DGmI4p7ApOofxIYGQy8e1A1WTiV7CSfnpp0lZK2YrOPaKUUm5E5x5RSik3oi1tpZRyIxkZp+2ONGkrpWzF5g1tTdpKKXvR7hGllHIj+iBSKaXciLa0lVLKjdg8Z2vSVkrZjM2ztiZtpZStaPeIUkq5EZvnbE3aSil7cXjYO2tr0lZK2Yp2jyillBvRpK2UUm7E5jlbk7ZSyl60pa2UUm5Ek7ZSSrkRu889oquxK6VsRUTSvaVxnooisstluygiL4rICBE54RJ/1OWYQSJyUER+E5FHXOKBIrLH2jdObuDXAU3aSilbEUn/lhpjzG/GmJrGmJpAIHAZWGTtfj9hnzFmhfO6UhnnaupVgCbApyLiYZUfD4QC5a2tyfXenyZtpZStiEPSvWVAMHDIGHM0lTItgDnGmChjzGHgIFBbRPyBvMaYTcYYA8wCQq7z9jRpK6XsJSMtbREJFZHtLltoCqftAHzp8r2PiOwWkWkikt+KBQDHXcqEWbEA6/PV8euiSVspZSsZ6dM2xkwyxtRy2SYlc74cQHPgKys0HigL1AROAWMSiiZTHZNK/Lro6BGllK1kwZC/psBOY8xpgIR/W9eaDCyzvoYBJVyOKw6ctOLFk4lfF21pK6VsJbMeRLroiEvXiNVHnaAlsNf6vBToICLeIlIa5wPHrcaYU0C4iNxvjRrpDCy53vvTlrZSylYys6UtIr5AI6C7S/hdEamJs4vjSMI+Y8w+EZkH7Adigd7GmDjrmJ7ADMAHWGlt10WTtlLKVjKzd8QYcxkocFXsqVTKjwRGJhPfDlTNjDpp0lZK2Yq+xq6UUm7E7q+xa9JWStmKzRvamrSVUvaiSVsppdyIJPsui31o0r5K1WpVKF++QuL3jz/6mICA5N84DawVyI7tO27oeoMHD+KnTT/x7arvyJEjB+fOnaNtuzas/m7NDZ1XZY28fj6MXfM0AH5FcxMXZ7jwdwQAPWpPIjYmLpWj0+eDH57Bzz8P0VdiibwUzbtdF3H897M3fN7bhba0bzPe3jlZtHBR2gUzkcPhwcKFC+jQoeNNva7KuIv/RvLc3eMBePrV+kReimbumI2J+z08HMTFxd/wdUZ2ms9vO07yeLdAerz3CENafHHD57xd6IPI21xERAR9+vbh4sULxMbG8vzzLxDcIDhJmb//PsPL/V7m0qUI4uJiGT78VWoF1mLjxo18/MlHREdHU6LEnYx8cyS5cuW65hqdn+rMzFkzadOm7TX7pk6byqpvviE6Jprg4Ib07dMXgPHjP2XZ8mUULVqU/Hfkp3KVKnR9pmvW/CGoVA2c3pKL/0ZS/m5/ft95ksjw6CTJfPqe3gx6/HP+OnqeRp2q0+r5+/HK4cH+LWF80GsZ8fEpT0Oxe/1R2rz4AAA93m3MfU3LYwzMfnMdP8zbi1/R3Lw6tx258nrj4elgbM9l7PkxtYno7E+H/N1moqKu0LJVSwCKFw/g/bEf8NG4j8idOzfnzp2jQ8cONKjfIMlfjGXLlxMU9BA9uvcgLi6OK1eucO7cOSZMHM/UKdPw9fVlypTJzJw5g169el9zTX9/f+65J5ClXy+lfr36ifGNGzdy7OhR5s6dhzGG3n16sX37NnLm9OHb775jwfyFxMXF0bpNKypXqZL1fzgqRSUqFKBfwxnExxuefrV+smXurFSQ+u2r0SdoCnGx8bz4yeM07FSdb2f/kuJ5H2hWkT/3nKZOq8qUq+nPszU+JV9BXyZs684v64/Q8InqbFt1kM/eWo/DIXj7emXVLboNm+dsTdpXu7p7JCYmhg8+eJ/tO7Yj4uDMmdP8888/FCpUKLFMtapVGTJ0KLGxsQQ3COauu+5i27atHDp0iE5Pdko8T82aNVK8bmi3UHr36U3dOnUTYxt/2sjGnzbSqnUrAC5fvszRo0eJiIigQYMG5MyZEyBJolfZY+1X+1JtMQMEBpehQqA/E7c534jO4ePF+TMRyZYd8nkboiNj+OvIeT7su5x2Lz/Imi93Ex9vOHcmgl/WHaHSvQH8uu0EA6aF4OnlwY+LD3Dwl78y/d7cjba0b3PLli3j33P/8tW8+Xh5edGwUTDR0dFJytSqdS+zZ81m3bq1DBw0gK7PdCVv3nw8+MCDjB49JoUzJ1WyZEnuqlSJb775JjFmjKFbt1Dat2ufpOzMmTNu+L5U5roS8d/fibjY+CQT7OfIaf1vJsKqmbuYPHh1mudL6NNOkFIi2r3hKM/Xmcb9j1Vg8OzWzHnvx1Rb7rcFe+dsneUvLZcuhePnVwAvLy+2bNnCyZPXzqh44uQJ/Pz8aNu2Ha1btWb//v3UqFGDnT//zNGjzv7FyMhIjhw5nOq1Qrt3Z/qMaYnfHwp6iIULFxIR4WyNnT59mrNnz3LPPYGsXfsDUVFRREREsG79uky8Y3Wj/jpyjgr3OCeCK3+3P0VLO+fI37nmT+q2qcIdhZzPNfLk96HInfnSdc5f1h+hfvtqOBxCvoK+VK9Til+3nqDInfk4fyaC5VN2sGLqTircUyxrbsqNZNYakbcqbWmn4fHHm9Grd0/atmtDpUqVKFOmzDVltm3dxrTpU/H09MLX15dRb4/Cz8+Pt0a+Rf/+rxAd42yFPd/3BUqVKp3itcqXK0/lypXZv38/AEFBQfz55yGe6OQcVeLr68s7o96lWrVq1K/fgJatQihWrBhVqlQlT+7cWXD36nqsW7Cfxp1rMuXnnvy67QRh1nC9owf+ZurQNYz+tjPiEGJj4vmw9zJOH7uQ5jk3LDpAlQdKMPWXXhgDE//3Lf+evsQjnWvSoX8QsTHxRF6K5q3OC7L69m55dh89Is4ly7JOXGwaHX3qukRERJArVy4iIyPp3OUpXhvxGpUru8/DyGCvEdldBXULWmtev+GMu2fPX+nOOdWqFXW7DK8tbTc1YsSrHDx0iOjoKFq0CHGrhK1UVnLXbo/00qTtpt57b3R2V0GpW5LNc7Ym7cw0ZOgQ1q1bi5+fH0uXfA3AgQMHeO31EURFRePp6cGwocOpXr164jEnT56kWfNm9O7dW1+OcWP/mxrCA49X4PyZCJ6p9gkAZasX4eUJzfHJnYO/jpznzU7zuRwehYeng/5TWlDhnmJ4eDpYNWsXX4zaAED9dlV5ckgdHB4ONi//nYkDvgWgSZea9HjvEf45cRGARR9vYfnUndlzs7c4u7e0dfRIJmoZEsKkiUkXcx4zdjS9evVm0cJF9OnTlzFjk7aQ33lnFA8//PDNrKbKAt/M+Jn/NZmdJNZ/SgiTBn5H1+qfsGHRfjr0DwKgXtsq5PD2pGv1TwgNnEDz7rUoWvIO8vr50OO9xrwcPINnqn5M/iK5uKfBfw++f5i7l+fuHs9zd4/XhJ2KLFgj8paiSTsT1ap1L/ny3ZEkJggRly4BcCn8EoULFU7ct3rNaoqXKEG5cuVuZjVVFti94Sjh/0YmiZWoWIBf1h8BYPt3h6jTujIAxkDOXDnw8HDg7eNJTHQcERej8C+Tn7Dfz3Lhn8sA7Fj9Z+IxKv0cDkn35o40aWexgQMH8d7o0TQIrs97o9/lxZdeApxvN06dOoVePXtlcw1VVjm89wxBzSsBUK9tVQqXcI7JXjd/H1ciollwqj9zj/Vj7uiNhJ+L5MTBf7mzUkGKlrwDDw8HD4VUSjwGoE7rykz9pRevfdWeQsXzZss9uYPMbGmLyBER2SMiu0RkuxXzE5HvROQP69/5XcoPEpGDIvKbiDziEg+0znNQRMbJDfThXHfSFpFnUtkXKiLbRWT75MmTUip2W5gzdw4DBwzk+zU/MGDAQIYNGwrAx598TOfOXZKdQErZw7tdFxPSuzYTt/fAN08OYqKd07beVbs4cXHxtC72Hh1Lv0+7fkH4l87PpfNXGNtzGcPntmPchmf568h54mKdx/z09W90KDWWZ2t8yo7Vhxg0s1V23totLQterqlvjKlpjKllfR8IrDHGlAfWWN8RkcpAB6AK0AT4VEQ8rGPGA6FAeWtrcr33dyMPIl8Dpie3wxgzCZgEOk57yZLFDB40GIAmjzRh+PBhAOzevZtvv13FmDGjCQ8PR8SBdw5vOnXqlJ3VVZno2G//0P+RWQAUL1+A+x9zztMe/EQ1tn5zkLjYeM7/HcHejceoWKsYpw6fY9Oy39i07DcAHu8WSHyc83+fiy5dL8sm7yD0ncY3+W6UixZAPevzTGAtMMCKzzHGRAGHReQgUFtEjgB5jTGbAERkFhACrLyei6eatEVkd0q7gCLXc8HbTeHChdm2bRu1a9dm85bNlCxZEoDPZn+WWObjTz7G19dXE7bN3FEoF+f/jkBEeGpoXZZO2AbAmWMXuKdBab777Bdy+npR+f7izP9gU5Jjct+Rk5BetRnRbh7gXHDh37+cz0YebF6JYwf+zp6bcgMZ6XkQkVCcLeAEk6xGZwIDfCsiBpho7StijDkFYIw5JSIJD6oCgM0ux4ZZsRjr89Xx65JWS7sI8Ahw7qq4AD9d70Xt6pVX+rF121bOnz9P/Qb16NO7D6+NeJ23R71FXGwcOby9eW3E69ldTZUFhn3Rhpr1SpOvoC9fHe/H9Fd/wCd3DkJ61wZgw8IDrJz+MwCLP9nKgOkhTN/bBxFYOf1n/txzGoC+HzalbI2iAMx6fS1hfzhfgW/9/P082LwScbHxhP8byainb+5CHe4kI73Frr0CKQgyxpy0EvN3IvJrapdO7hKpxK9Lqq+xi8hUYLox5sdk9n1hjHkirQvc7t0jKnn6GrtKTma8xv7nn2fTnXPKlCmQ7uuJyAjgEtANqGe1sv2BtcaYiiIyCMAY87ZVfhUwAjgC/GCMqWTFO1rHd0/vtV2l+iDSGPNscgnb2pdmwlZKqZtNMvBPqucRySUieRI+A42BvcBSoItVrAuwxPq8FOggIt4iUhrnA8etVldKuIjcb40a6exyTIbpG5FKKXvJvOHXRYBFVh+5J/CFMeYbEdkGzBORZ4FjQFsAY8w+EZkH7Adigd7GmISVnnsCMwAfnA8gr+shZEJFlFLKNjLrTUdjzJ/ANctNGWPOAsHXHgHGmJHAyGTi24GqmVEvfbnmOmzYsIFHH2vKI00eYfLkySmW27NnD1WrVWHVqlWJsYsXL/Liiy/w2OOP8nizx9i1y/lwasyY0YS0bMHAQQMSyy5duoTZs2dl3Y2oG+ZwCJN39uTtr50jf8pWL8InP3Vj2u7evLW0E755vJM9bs7hl5i2uzdTfu6ZuPyYq/b9glhrXidfAV8Aqj54J1N/6cWErd0JKOsHQO58OXn3m85ZdGfuK7O6R25VmrQzKC4ujjdHvsHECZP4eunXrFixnIMHDyZbbuzYMQQFBSWJv/32Wzz00EMsX7aChQsWUaZMWcLDw/l51y4WL1pCXFw8v//+O1euXGHR4sV06NDxZt2aug6tX3iAoy7D71KabyQ5L9WfznN3j6f7vROTxAsVz0tgo7L8dfR8YqxdvwcZ3noOkwevpnnPewF4alhdPn9rfebekA3o3CMqiT17dnNniTspUaIEOXLkoOmjj/L9D99fU+7zzz+jUaNGFPArkBi7dOkS23dsp3XrNgDkyJGDvHnz4nA4iImJwRhDVNQVPD09mTZtKk8++SReXrq69q2qUEBe7n+sAsun7EiMpTTfSEb0eb8pE/+3yjlJiSU2Jg5vHy9y+noRFxNPsTL5KRSQN/Fa6j92X25Mk3YGnT59hqL+RRO/Fy1ShDOnT19V5jSr16ymffsOSeLHjx/HL78fQ4YMplXrVgwbPpTLly+TK1cuGjdqRKvWrQgIKE6ePLnZu3cvwQ2S7TZTt4g+HziTq3EZ1ZrSfCNXMwbe+7YzE7f34PFugYnxB5tV5O8TFzm0O+nfqS/e3sArk5rT5sUHWPTxFp4b2ZCpw9ZkwV3ZgGRgc0OatDPIJDcm/qqf2G+Pept+L/fDw8MjSTwuLo79B/bTvkMHFi5YiI+PL1OmOPvEn332ORYtXMSA/w1g3Efj6NO3L/Pnf8VLL7/EhAnjs+x+1PV54LEKnDsTwe87TyWJpzTfyNX6BE0hNHACA5rOJqT3fVR/uCTePl48OaQu04df+5vbwV/+otcDk3mpwXT8y+Tnn5PhiAjD57RlyOzW5C+sc9gksHnO1tEjGVW0SBH+OvVX4ve/Tp+mcOHCScrs27eXfq/0A+DcufOs37AeD08PalSvQZEiRahR3flAunHjxolJO8H+A85FfUuVLMXbb7/F7Fmf0e+Vlzly9AilSpbKwjtTGVE16E6Cmlfk/kfLkyOnJ755vRkyuzUjn1qQ7HwjVzt7KhyA839H8OOiA9xVuzjh5yLxL30HU39xzvxYqHheJu3sQc/ak/j39KXEY58aWpfX2s/jhY8fY8arP1C01B20ev5+pg7VljfYfxEETdoZVLVqNY4eO0pYWBiFCxdm5YoVvPvee0nKfPft6sTPgwcPom7dejQMbghA0aL+HD58mNKlS7N582bKlk06l/ZHH43jtRGvExsbS3xcPAAOcXAl8koW35nKiMmDVzN5sPO/c826pWj/ShAjn1qQ4nwjrnL6eiEOIfJSNDl9vajVuCyzXl/L4b1naFnk3cRycw6/RPdaE7lw9nJirEmXmmxe/juXzl8hp68X8fGG+HhDTl999nG70KSdQZ6engwZMpRuoc8RHx9Py5atKF+uPHPmzgGgw1X92FcbMngI/xvQn5iYGIoXL8HIN/8b0rl6zWqqVq2W2HKvUbMmLUKaU6FCRSpVqpR1N6UyTXDHasnON1LAPw/9p7Rg4GOfkb9Ibt5Y5BwV5OHpYM0Xu9m66toRSFfz9vHikS5380rjmQDMG7uJ1xd0ICY6jjc6fpVFd+R+3HVxg/RKde6RzKBzj6jk6NwjKjmZMffIqZMX051z/IvldbsMry1tpZSt2LxLW5O2Uspe3PVNx/TSpK2Ushd752xN2kope7H5c0hN2kopm7F5p7YmbaWUrdg7ZWvSVkrZjM0b2pq0lVI2Y/OsrUlbKWUr9k7ZOsufUspmMms+bREpISI/iMgBEdknIi9Y8REickJEdlnboy7HDBKRgyLym4g84hIPFJE91r5xcgOzWmlLWyllK5nYOxIL9DPG7LRWZd8hIt9Z+943xoxOel2pDHQAqgDFgNUiUsFa3Hc8EApsBlYATbjOxX21pa2UUskwxpwyxuy0PocDB4CAVA5pAcwxxkQZYw4DB4HaIuIP5DXGbDLOyZ5mASHXWy9N2kopW8nIGpEiEioi21220OTPKaWAu4EtVqiPiOwWkWkikt+KBQDHXQ4Ls2IB1uer49dFk7ZSymbSv3aNMWaSMaaWyzbpmrOJ5AYWAC8aYy7i7OooC9QETgFjXC58NZNK/Lpon7ZSylYyc8SfiHjhTNifG2MWAhhjTrvsnwwss76GASVcDi8OnLTixZOJXxdtaSul7CWTFom0RnhMBQ4YY8a6xP1dirUE9lqflwIdRMRbREoD5YGtxphTQLiI3G+dszOw5HpvT1vaSilbycSpWYOAp4A9IrLLig0GOopITZxdHEeA7gDGmH0iMg/Yj3PkSW9r5AhAT2AG4INz1Mh1jRwBXblGZRNduUYlJzNWrgm/cCXdOSdPvpxu9y6OtrSVUrZi87fYtU9bKaXciba0lVK2cgNviLsFTdpKKXuxd87WpK2Usheb52xN2kopm7F594g+iFRKKTeiLW2llK3Yu52tSVspZTM6ekQppdyJvXO2Jm2llL3YPGdr0lZK2YzNs7YmbaWUzdg7a2vSVkrZir1TtiZtpZTN2HzwiCZtpZTN2Dxr6xuRSinlRrSlrZSyFZs3tLWlrZRS7kRb2kopW7H7a+xZvrCv+o+IhBpjJmV3PdStRf9eqIzQ7pGbKzS7K6BuSfr3QqWbJm2llHIjmrSVUsqNaNK+ubTfUiVH/16odNMHkUop5Ua0pa2UUm5Ek7ZSSrkRTdo3iYg0EZHfROSgiAzM7vqo7Cci00TkjIjsze66KPehSfsmEBEP4BOgKVAZ6CgilbO3VuoWMANokt2VUO5Fk/bNURs4aIz50xgTDcwBWmRznVQ2M8asB/7N7noo96JJ++YIAI67fA+zYkoplSGatG+O5Gaw0bGWSqkM06R9c4QBJVy+FwdOZlNdlFJuTJP2zbENKC8ipUUkB9ABWJrNdVJKuSFN2jeBMSYW6AOsAg4A84wx+7K3Viq7iciXwCagooiEiciz2V0ndevT19iVUsqNaEtbKaXciCZtpZRyI5q0lVLKjWjSVkopN6JJWyml3IgmbaWUciOatJVSyo38Hx77C+jkJW04AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nb_matrix = confusion_matrix(y_test, nb_pred) \n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in nb_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in nb_matrix.flatten()/np.sum(nb_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(nb_matrix, annot=labels, fmt='', cmap='Purples')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "d7ce3fed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.99      0.97      0.98     20079\n",
      "         1.0       0.97      0.99      0.98     20079\n",
      "\n",
      "    accuracy                           0.98     40158\n",
      "   macro avg       0.98      0.98      0.98     40158\n",
      "weighted avg       0.98      0.98      0.98     40158\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "nb_report = classification_report(y_test, nb_pred) \n",
    "print(nb_report)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a38bd9e8",
   "metadata": {},
   "source": [
    "## Linear SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "9ee4e451",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5349120972159969"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc_pred = lsvc_model.predict(X_test)\n",
    "lsvc_score = lsvc_model.score(X_test, y_test)\n",
    "lsvc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "5fbedae5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAD4CAYAAAAn3bdmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAyEUlEQVR4nO3dd3gUxRvA8e97CSWh9xJ6L9KRJh2kSBUBKVIUqSoqotIEAemCggL+gvSOSleQIkV6lQ6C0kFAAQklIZfM74/bxAskJIEE2OP9PM8+7M3OlonxvcnsFDHGoJRSyh4cT/oBlFJKxZwGbaWUshEN2kopZSMatJVSykY0aCullI14x/cNenqn1O4p6j6fXz3+pB9BPY2Sp5NHvUQXSR7jmPONufHI93vctKatlFI2Eu81baWUepw8vSaqQVsp5VG8xXYtHrGiQVsp5VEcnh2zNWgrpTyLNo8opZSNOLR5RCml7ENr2kopZSPapq2UUjbipc0jSillH9o8opRSNqLNI0opZSNa01ZKKRvRLn9KKWUj3p4dszVoK6U8izaPKKWUjTjw7Kq2Bm2llEfR3iNKKWUj2jyilFI2ojVtpZSyEV0EQSmlbESbR5RSyka0eUQppWzE07v8efpfEkqpZ4xDYr5FR0SmiMhlETnoljZfRH6ztlMi8puVnkNE7rgd+8btnFIickBETojIOBFXw7uIJLKud0JEtotIjuieSWvaSimP4hW3Fe1pwNfAjLAEY8yrYfsiMhr41y3/H8aY4pFcZyLQCdgG/ATUAVYAHYBrxpg8ItICGAG8Gsn54bSmrZTyKA4kxlt0jDEbgauRHbNqy82BuQ+6hohkApIbY7YaYwyuL4DG1uFGwHRr/3ugRlgtPOryKaWUB4lN84iIdBKRXW5bp1jcqhJwyRhz3C0tp4jsFZENIlLJSvMDzrnlOWelhR07C2CMceKqtad50E21eUQp5VFiUxM1xvgD/g95q5ZErGVfBLIZY/4RkVLAYhEpDJFW6Y3174OORUqDtlLKozyOviMi4g00AUqFpRljgoAga3+3iPwB5MNVs87idnoW4IK1fw7ICpyzrpmCKJpjwmjziFLKozhEYrw9gprAUWNMeLOHiKQTES9rPxeQF/jTGHMRCBCRclZ7dVtgiXXaUqCdtd8U+MVq9466fI/y1Eop9bRxxGKLjojMBbYC+UXknIh0sA614P4XkJWB/SKyD9dLxS7GmLBac1fgW+AE8AeuniMAk4E0InIC6AH0iu6ZtHlEKeVR4rJ5xBjTMor09pGk/QD8EEX+XcBzkaQHAs1i80watJVSHiWaHnO2p0FbKeVRPDtka9BWSnkYT39Rp0FbKeVRPLx1RIO2UsqzePosfxq0lVIexbNDtgbtSPmmTkXnVUsBSJYxPSYkhJtX/gFgXPnqhAQHP/I9uq5dTsIkSRhbrhoAWUoVp8HIz5hYo/4jX1vFj4JlK5Mvd67wz+M/H0aWzJkizVui8ovs3bj6ke7X69Mh7Nj7G8mSJMHhcND/ox6UKHpfrzF1D10E4Rl0++o1vijtmuulVv9eBN28yYYxX4cfd3h5ERoS8sj3SZo+LQXq1OToyjWPfC0V/xInSsSSOdMe6z0/6t6NOjWqsWnbDvoPG8WyudOjP+kZJx5e19agHUOvTp7A7WvX8CtelPN79xEUcDNCMO/52xYmN2rBtdNnKNmqORXf6YxXgoSc2bGLhW9/gAkNve+a60d/RY0+Pe8L2uJwUG/Yp+SuXBGvRInYMnES2yZNQ0R4edwoclV+gaunTiMOBzunzmL/wqWP5WegIrp1+zbdPujNjYAAnE4n73btSM0qlSLkufz337zfZwA3b94iJCSET3v1pHSJYmzatoOv/Cdz924wWbNkZlj/PiTx9Y3yXs+XKMaZs64R01Nnz+OHpT8C0LRRA9q3as7tO3d4r3d//rp8mdCQULp1aM9LtWrEX+GfYp4dsjVox0q6vLn5X61GmNBQavWPfLRp+gL5KN68CV9Xqk2o00mTrz6nZKvm7J417768p7ftoEjj+uSuWomggIDw9DJvtOHOvzcYW746XgkT8vbGn/l99TqylCxGqhzZGF28AknTp+PDgzvYOXVWvJVXRRQYFESjVu0ByOKXibHDBjN+1FCSJk3C1evXefX1ztSoXDHC4I7lK1dTsVwZur7RjpCQEO4EBnH1+nUmTpnO1PFf4uvjg//0WUydPZ+3O74e5b1/+XUz+fLk4uCRoyxc9hMLpvljjKF5+06UKVWcs+cvkD5tWvy/HAVAwM2b8fqzeJpp84gKt//7JZHWmN3lrV4Fv5LFeHfbOgAS+CTm5pW/o8y/Zujn1OzTkx97DwhPy/9idTIVKUzRJo0A8EmRnLR5c5HjhfKuZzCGgEuX+WP9r3FQKhVT9zaPBDudjJnwP3bu3YdDhEtXrvD3P1dJl/a/6ZCLFCpIn8HDcDqd1KxSmYL587Lu172c+PMULTt0Db9O8SKFI73nyHETmDh5OqlTpWRIv95s3bmLmlUr4+vjA8CL1aqwa+8+KpUvy4ix4xn11QSqVXyB0iWKxd8P4imnvUdUuLu3boXvhzidiOO/bvzeiRO7dkTYNXMuK/oOitE1T6zbSO2Bfche9vn/EkVY9N5H/L7qlwh5C9at/fAPr+LcshWruHrtOgtnTiaBtzfVGzYl6O7dCHmeL1mcWf7j2bBpCx8NGEyHNi1JniwZL5QtzZghA6O9R1ibdpgtO3ZFmi9n9mwsnDGZDZu3Mnr8N7xQtswDa+6ezLNDtucPHoo3106dwc+qzfiVKEbqnNkBOPHLBoo2aUTSdGkB8EmVklTZsj7wWmuHjaZqz+7hn4+tWkuFzh1weLu+U9PmzU1CX19Obt5KkSYNEBGSpk9H7ioV46NoKoYCbt4kTepUJPD2ZtuuPZy/+Nd9ec5f/Is0qVLS/OWGvNKwPoeO/k7xIoXZs+8Ap6026juBgZw8fSZG93y+ZDHWbPiVO4GB3L5zhzXrN1K6RDEuXfkbn8SJaPRSbTq81pLDx36P07LaiUjMNzvSmvZD2r9wKaXatOD9Xb9ydtcervx+AoBLR46xsv9ndFyxCHE4CA0OZmH3nlw7czbKax1dsZpbVpdCgB2TZ5A6Rzbe37kBEeHm3/8wrUlrDixcSt7qVei5bytXjv/BmR27uHPjRryXVUWuQd1adO3xMU3adqBgvrzkypH9vjw7du9l8sw5eHt74+vrw4hP+5E6VSqGDehLj76fctfqPvpel47kzJ4t2nsWLpCfJvXr0qxdR8D1IrJQ/nz8unU7I8dNwCGCt7c3n/bqGbeFtRGbxuIYk2jm235kPb1Txu8NnjEJkyTh7q1b+KZORfetvzC+cm0CLl1+0o8Va59fPR59JvXsSZ7ukWPu+vRZYhxzql4+Z7sYrzVtm+mwdD6JU6TAO2EC1gwZZcuArVR88rJdGI4dDdo2oyMmlXowD4/ZGrSVUp7F00dEau+RWGo+6Ws+vXCcnr9tCU/LVPQ53t60ig/2buaNxfNIlCxZhHNSZs3CkOvnqNLj7fA0v5LF+GDvZnod3UOjL0aEp+eqVIH3dmxgRODfFG3SMP4LpOLdjYAAun/cjzpNW1G3WWv27j/Ie73706hVexq1ak/1hk3DB+0EO518/OlnNGjRlrrNWvO/qTOf7MPbUFz2HhGRKSJyWUQOuqV9KiLnReQ3a3vJ7VhvETkhIsdEpLZbeikROWAdG2ct8IuIJBKR+Vb6dhHJEd0zadCOpV0z5jCpXtMIac3/N46f+gxkdIkXOLB4eYTuewANRw+9b6j6K+PH8H3X9xheoCTp8uaiQJ2aAFw7c475Hbqxd+738VsQ9dgMGT2WSuXLsvL7OSyZM43cObPz5bBBLJkzjSVzplGrWhVerFYFgJVrfuHu3WCWzZvBwpmTmb9oCecuXHzCJbCXuFzYF5gG1Ikk/QtjTHFr+wlARArhWvC3sHXOhLDV2YGJQCdcK7TndbtmB+CaMSYP8AUwgmhE+9wiUkBEPra+HcZa+wWjO89T/fnrFm5fvRYhLV3+PPy5cTMAv69ZR9GXG4QfK9ywHldPnuLS4aPhackyZiBxsmSc3rYTgF0z51G4YT0Arp0+w8UDh6Ideans4ebNW+zcu4+mjVzvIhImSEByt7/EjDGsWLOO+rVdX9oiwp07d3A6nQQGBpEggTdJkyR5Is9uVxKLLTrGmI3A1WgzujQC5hljgowxJ3GtvF5GRDIByY0xW42ru94MoLHbOWGzgH0P1AirhUflgUFbRD4G5uEq3w5gp7U/V0SiXer9WfHXoSMUbuD6C6lY08akyOoHQEJfX6p99C6rBkX88kzhl4nr5y+Ef/73/AVS+EU+xaeyt7PnL5A6ZUp6DxxK49av0/ez4dy+cyf8+K69+0iTJhU5rAFYtWtUw8fHh4p1G1OtwSu80bolKVMkf1KPb0sOkRhvItJJRHa5bZ1ieJu3RWS/1XySykrzA9wHZJyz0vys/XvTI5xjjHEC/wJpeIDoatodgOeNMcONMbOsbThQxjoWKfcfxP7Qu1Fl8xjz33ybCt3e5L3t60mULCkhd10DJmp92ptfv5wQYfg7RLFatPZm90jOkBAOH/udlk0bs3j2VHwSJ8Z/2n+TfC1ftYb6tWqGf95/6DAOh4NfVyxm7ZLvmDJ7HmfPnX8Sj25bsalpG2P8jTGl3Tb/GNxiIpAbKA5cBEa73fpe5gHpDzonStH1HgkFMgOn70nPZB2LlFVwf3g2BtdcOXacSXWbAK4h5wVfqgVAtjKlKNqkEfWGD8InZQpMaCjOwCD2L1xKSr/M4een8MvMv9pu6ZEypk9HxvTpKPaca0KoOjWq4T/dFbSdTier121g4YzJ4fmXr1xNpQplSeDtTZrUqShZrAgHjhwlaxa/SK+v7hdN68IjM8ZccrvXJGC59fEc4D5nRRbggpWeJZJ093POiYg3kIJommOiC9rvAWtF5Dj/VfuzAXmAt6M66VmTNF1abl75GxGhZp8P2fq/qQBMqBr+Ujl8MYXNEyYBEBRwk2xlS3Nm+y5Kt2nBpvEx+YJXdpMubRoyZkjPn6fOkCtHNrbu3EXunDkA1+RPubJnJ2OG9OH5M2XMwPade2hUtzZ3AgPZd/Aw7Vo2f0JPb0/xPTWriGQyxoTVsl4GwnqWLAXmiMgYXJXdvMAOY0yIiASISDlgO9AW+MrtnHbAVqAp8IuJZpj6A4O2MWaliOTD1Rzih6sqfw7YaYx59KVbbKj1rG/JXaUiSdKmod+pQ6waOJyESZPwQtc3ATiweBk73f78jcoPb/egxeQJePv4cGzlao6ucC1NlbV0Cdp9PwvfVCkpVL8OtQb05vNi5eO1TCp+fdLzfXr2H0hwsJOsfpkZ1r83AD+tWku92jUj5G3drAm9Bw2l/qttMECTBi9RIG+eJ/DU9iVxGLVFZC5QFUgrIueAAUBVESmOqxnjFNAZwBhzSEQWAIcBJ/CWW5zsiqsnig+wwtoAJgMzReQErhp2i2ifSeceUU+Czj2iIhUHc4/sz54jxjGn6OlTthuJoyMilVIeJb7btJ80DdpKKY/i4TFbR0TGhjgcvL9zI28sca33mLlYEd7ZvJr3d/3Ku9vWkfX5kved450oEd23rqXH7k303LeVWgN6hx8r+kojeu7bysi7V8lSqnh4eo4KZemxZzPvbv2FNLlzApA4RQo6/vRD/BZQPbTeg4ZSvlZ96r/aJso823fvoVGr9tRr/hqvdfrvPf70uQuo/2ob6jV/jWlzFoSnj/pqAg1atuOjAYPD0xb/tJLpcxegoiau/tcx2uxIg3YsVOrelUtHj4V/rjd8IKsHj+CL0pX4eeBQ6g+/f4kxZ1AQ39RsyJhSFRlTqhIFatcgW9nSgGtQzvRmbTj565YI51R5/21mNG/DT58MokIXV3f4F/t9yNrhY+KxdOpRNKn/Et+OGx3l8RsBAQwcMYaJY4bz44JZjB3uCsS/n/iT7xYv47vpk1gyZxrrN23m1JmzBNy8yd79B1k2dzohIaEcO/EHgYFBLFq2glbNmjyuYtmSp69co0E7hlL4ZabgS7XYMcVtAh9jwieHSpw8eZR9rcMG13glSIDDOwFYL38vH/09fMUbdyHBwSTw8SGhjy8hwcGkyZWD5JkzhQ+VV0+f50sWJ0XyqEcuLlu5mherVSZzxowApEntGkT3x6lTFCtSGJ/EifH29ub5kiVYvX4jIg6Cg4MxxhAUFIS3tzffzpxDmxZNSeCtrZoPEpsRkXak//VjqNGYYSzv1Z/EbvNGLOnRm44//UCDkYMRh4OvK0W+8K44HLy3YwNp8+Rky8RvObNj9wPv9cuIL2j6zZcE3wlkbrvO1B85mJ8HDInT8qjH69SZszidTtp0fptbt2/TtkUzGterS77cufhyoj/Xrv9L4sSJ2LhlK88VLEDSJL7Uql6Vxq1fp/zzpUiWNAkHDx95ZhfrjQ1HfHfUfsI0aMdAwXq1uXn5Cuf37IuwmG75zh1Y+kFfDixaSrGmjWk26Sv8aze+73wTGsoXpSuROEUK2v8wi4yFC/LXoSNR3u/CvgN89cKLgGuq1hsX/wIRXpszhZDgYJZ92I+bl6/EeTlV/AkJCeHQ0WNMmzCWwKAgWrzRhWLPFSZ3zhy82fY13nj7fXx9fcifNw9eXq6J4Tq2bU3Htq0B6PvZcLp3eZPvFi9j0/Yd5M+Tm24d2j/BEj29xMPbDzy8eHEjR4WyFGpQlz4n9tN69mTyVKtMy+n/o3TbFhxYtBSAfd8vJlskLyLdBf77L39s2ET+2jVifO8afXqy5rOR1PrkY34eOIw9cxZQ8Z3Oj1Qe9fhlTJ+OSuXL4uvjQ+qUKSldohhHj7uaxpo1qs+iWVOY7T+elMmTkz1rlgjnhq2sniNbVhb/tJKxwwZz/I+TnHrAYtHPMn0RqVjRdxCf5SjM0DxFmd26AyfWbWRuu87cuPBXeM07T/XK/H38z/vOTZI2DYlTpADAO3Fi8taowuVjMRtYUrptK478tIo71/8lga8vJjQUExpKQl/fuCuceixqVKnErr37cTqd3AkMZP/Bw+TOkQOAf6ypfi/89Rer1m0In6Y1zNhvvqV75zdxOp2EhLim/HE4hMDAwMdaBrvw9BeR2jzyCL7r8i6NxwzH4e2NMyiQ77q+C0DyTBlp5j+OyQ2akzxTRlpMmYh4eeFwCPu+X8yRH38G4LlG9Wk8dgRJ06Wlw9IFXNh3gEkvvQJAAh8fSrdtiX+dlwHY+OV42i2YQcjdYGa9FuUEi+oJ6dF3ADt2/8a169epXO9l3unUAafTCUDLVxqTO2cOKlUoS8NW7XGI0LRRA/LlyQXAOx/35fq/N/D29mLARz0ivNBcs34jRQoVIEO6tACUKFKYBi3aki9Pbgrky/v4C2oDdq1Bx5QOY1dPhA5jV5GKg2HsJ5/LF+OYk/Pg77aL8FrTVkp5FC/tPaKUUvbh6c0jGrSVUh7Fw2O2Bm2llGfRoK2UUjYSl4sgPI00aCulPIq+iFRKKRvR5hGllLIRT+89osPYlVIeJS6HsYvIFBG5LCIH3dJGichREdkvIotEJKWVnkNE7ojIb9b2jds5pUTkgIicEJFxYn2ziEgiEZlvpW8XkRzRPZMGbaWUR4njCaOmAXXuSVsNPGeMKQr8DvR2O/aHMaa4tXVxS58IdALyWlvYNTsA14wxeYAvgBHRPZAGbaWUR4nLmrYxZiNw9Z60VcYYp/VxG5DlvhMjPI9kApIbY7Ya17whM4DG1uFGwHRr/3ughkTzbaJBWynlURxeEuNNRDqJyC63rVMsb/cGsMLtc04R2SsiG0SkkpXmB5xzy3POSgs7dhbA+iL4F0jzoBvqi0illEeJzYtIY4w/4P+Q9+kLOIHZVtJFIJsx5h8RKQUsFpHCQGQPFDap1YOORUqDtlLKszyGftoi0g6oD9SwmjwwxgQBQdb+bhH5A8iHq2bt3oSSBbhg7Z8DsgLnRMQbSME9zTH30uYRpZRniedVEESkDvAx0NAYc9stPZ2IeFn7uXC9cPzTGHMRCBCRclZ7dVtgiXXaUqCdtd8U+MVEM1+21rSVUh4lLvtpi8hcoCqQVkTOAQNw9RZJBKy27rXN6ilSGRgkIk4gBOhijAmrNXfF1RPFB1cbeFg7+GRgpoicwFXDbhHdM2nQVkp5Fq+4a0AwxrSMJHlyFHl/AH6I4tgu4LlI0gOBZrF5Jg3aSimPohNGKaWUnXj4MHYN2kopj6I1baWUshOtaSullI1oTVsppexD4rD3yNNIg7ZSyrNo84hSStmHeHZFW4O2UsrDaE1bKaXsQ7v8KaWUnWhNWyml7EN7jyillJ1o84hSStmINo8opZR9xOV82k8jDdpKKc+izSNKKWUf+iJSKaXsxMObRzz7K0kp9cwRh8R4i/ZaIlNE5LKIHHRLSy0iq0XkuPVvKrdjvUXkhIgcE5HabumlROSAdWyctcAvIpJIROZb6dtFJEd0z6RBWynlWeJ2NfZpQJ170noBa40xeYG11mdEpBCuhXkLW+dMCFudHZgIdMK1Qntet2t2AK4ZY/IAXwAjonsgDdpKKc/ikJhv0TDGbMS1Srq7RsB0a3860NgtfZ4xJsgYcxI4AZQRkUxAcmPMVmOMAWbcc07Ytb4Hakg03V/ivU378xun4/sWyoa6JMn6pB9BPYW+MTce+RqPoctfBmPMRQBjzEURSW+l+wHb3PKds9KCrf1708POOWtdyyki/wJpgL+jurm+iFRKeZZY9B4RkU64mi3C+Btj/B/yzpF9W5gHpD/onChp0FZKeZZY1LStAB3bIH1JRDJZtexMwGUr/Rzg/idkFuCClZ4lknT3c86JiDeQgvubYyLQNm2llGeJ2xeRkVkKtLP22wFL3NJbWD1CcuJ64bjDakoJEJFyVnt123vOCbtWU+AXq907SlrTVkp5Fkfc1UVFZC5QFUgrIueAAcBwYIGIdADOAM0AjDGHRGQBcBhwAm8ZY0KsS3XF1RPFB1hhbQCTgZkicgJXDbtFdM+kQVsp5Vni8EWkMaZlFIdqRJF/CDAkkvRdwHORpAdiBf2Y0qCtlPIsHj4iUoO2UsqzeHlFn8fGNGgrpTyL1rSVUspGNGgrpZSNaNBWSikbicMuf08jDdpKKc+iQVsppWxEm0eUUso+RGvaSillI1rTVkopG9GgrZRSNqJBWymlbESHsSullI1oTVsppWxEg7ZSStmIdvlTSikb0Zq2UkrZiAZtpZSyEQ/vPeLZjT9KqWdPHK3GLiL5ReQ3t+2GiLwnIp+KyHm39JfczuktIidE5JiI1HZLLyUiB6xj46xV2R+KBm2llGeJo6BtjDlmjClujCkOlAJuA4usw1+EHTPG/OS6rRTCtZp6YaAOMEFEwqr9E4FOQF5rq/OwxdOgrZTyLA5HzLeYqwH8YYw5/YA8jYB5xpggY8xJ4ARQRkQyAcmNMVuNMQaYATR+yNJp0FZKeZhY1LRFpJOI7HLbOkVx1RbAXLfPb4vIfhGZIiKprDQ/4KxbnnNWmp+1f2/6Q9GgrZTyLA6vGG/GGH9jTGm3zf/ey4lIQqAh8J2VNBHIDRQHLgKjw7JG8jTmAekPRXuPKKU8iyPOu/zVBfYYYy4BhP0LICKTgOXWx3NAVrfzsgAXrPQskaQ/FK1pK6U8izhivsVMS9yaRqw26jAvAwet/aVACxFJJCI5cb1w3GGMuQgEiEg5q9dIW2DJwxZPa9pKKc8Sh4NrRMQXeBHo7JY8UkSK42riOBV2zBhzSEQWAIcBJ/CWMSbEOqcrMA3wAVZY20PRoK2U8ixxOPeIMeY2kOaetDYPyD8EGBJJ+i7gubh4Jg3aSinPosPYlVLKRhyePYxdg7ZSyrPo1KxKKWUj2jyilFI2EvOufLakQfseBUuVI1+e3OGfx38xiiyZM0eat0SFKuzdsuGR7ter/0A2b9vB2uWLSJgwIVevXadp63b88tNDd+NU8ShJ6tS8t3YpAMkzZsCEhBBw5W8AhpepRkhw8CPfo8e6H0meKSPOwECCbt5ixhvduPT7iUe+7jMj7gfXPFU0aN8jcaJELJk/+7He08vLwfeLl9KqedPHel8Ve7euXmVIiYoA1B/Qm6CbN1k9+qvw4w4vL0JDQqI6PcamtH6TM7v3UrFje5qM+oyJjVo88jWfGfoi8tl26/Ztur3fkxs3AnA6nbzbrQs1q1WJkOfylb95/+M+3Lx1i5CQED7t8zGlS5Zg09ZtfDXRn7vBwWTN4sewgf1J4ut73z3atWrB9Nlzad6k8X3Hvp0+kxWr1nA3OJgXq1Wle1fXfDbj/SezbMVKMmXIQKqUKSlcqAAd2r4WHz8CFY12Uydy6+o1spYoypk9+wgKuBkhmH9yYBsT6jfnn9NnKNP6Vap374JXwgSc3L6Lud16YEJDo7z28Y2bqfFeNwCajBxM4bovgjH89Nkodi9YSPKMGeg4fxqJkyfD4e3N3K7vc2LT1sdS7qeWNo88WwKDgmj0amsAsvhlZuzIYYwfPZKkSZNy9dp1Xm33BjWqVsZ9DvPlK36mYoVydH3zDUJCQrgTGMjVa9eZOGkKU/83Hl8fH/ynTmfqzDm83fnN++6ZKWNGSpYozpIfV1CtcqXw9E1bt3H6zFm+nzUNYwxd3/uAnbv3kDhxYlat/YXFc2fiDAmhScu2FC5UIP5/OCpK6fPl4cuaDTGhodQf0DvSPBkL5KP0q00Y+cKLhDqdtBw/hjKtX2X7zLmR5gco2qAu5w8cokSThmQtXoTPilUgado09N65nuMbN1OmVTMO/7yWFUM/RxwOEkZSKXjmaPPIs+Xe5pHgYCdjvp7Izj17cYhw6fIV/v7nH9KlTRuep0jhgvQZ+BlOp5Oa1apSMH8+1u3+lRMnT9Ky/Zvh1yleNOoBUV3eaE/X93pStdIL4Wmbt25n89btNG7hqkHfvnOHU2fOcuv2bWpUrULixIkBqFa5Ylz+CNRD2PPd4gfWmAEK1KhKtlLF6b1zPQAJfHwIuHwl0rxvzP6W4Dt3+OfUGea/8yE1e7zNzrnfY0JDCbh8hd83bCbH8yU5tXMPbadMwCtBAn5bvJxz+w7EddHsR3uPPNuWrVjJ1WvXWDh7BgkSeFP9pUYE3b0bIc/zpUoy69v/sWHTZj7qN4AObV8jefLkvFC2LGOGfxaj+2TPlpWC+fOyYtWa8DRjDJ3eaEeLpk0i5J02a86jF0zFqbu3boXvhzidiFtf4QTWlysibJs+h8V9BkZ7vbA27XBRBKITv25hdOU6FKlXm9dn+rNq1LgH1tyfCR7ePOLZpYsDATdvkiZVKhIk8Gbbzl2cv3jxvjznL1wkTepUNG/SmFcaN+TQ0WMUL/Ice/bt4/QZ15zod+4EcvL0gxa9gC5vvs6UGf/V8itWKMcPS5Zx6/ZtAC5dvsw/V69SskRx1m38laCgIG7dvs36TZvjsMTqUf1z6gxZSxYHIGuJYqTNmR2AY2vXU7JpY5Klc/2V5psqFamzZY3qMhEc37iZUq++gjgcJE2bhryVK3Bqx25SZ8tKwOUrbPp2OpsnzyRbyWLxUiZbcUjMNxvSmnY0GtStQ9d3e9CkVVsK5s9Hrpw57suzY9duJs+Yhbe3N76+PowY/CmpU6di2MD+9Ojdj7tWN7D3unUhZ/bsUd4rb+7cFCqYn8NHjgFQsXw5/jh5ihbtOgDg6+PDqCGDKFq4ENWrVKLhq63xy5SJ5woVJFnSpHFfePVQ9v6whHJtW9J37yZO79wT3l3v4pFjLOk3mO6rFiMOByHBwcx7qydXz5yN5orw26Jl5Cpfhn77toAxLPyoPzcuXaZc21a8+GF3QoKDCbp5i2ltO0d7LY/n4b1HxLVkWTy6/W883+DZdOv2bZL4+nLnTiCtO3Ri8Cd9KFzQPi8juySJWQ1TPVu+MTceufobsmRCjGOOV6Nutqtua03bpvoPHsqJP08SdPcuL9evZ6uArVS8smmzR0xp0Lap0cNi9oJTqWeOvohUMdX708GUr16b+k3/G7125NjvNG/7Bo1ebU2TVm3Zf/BQhHMuXPyLEhWqMHnGrMf9uCoOtZk8npGX/uCTA9vC0/yKPsdHW9bwyf6tdFs6n8TJkgHg8Pam3bRv+GT/VgYc3kntXj3CzynVvAn99m2h/8HtNBkxKDy9fLtWjLr8J333bqLv3k280KHt4yuc3cRiNXY70qAdh5o0qMe348dGSBv15Ve81elNlsyfzbtdOzPqy68iHB/2+RdUeqH843xMFQ+2TpvNV3Uids1s8+3XLOo1gMFFy/PbomW8+OG7AJRq9jLeiRIxuGh5hpaqTOXOr5MmezaSpE7NK6MG82WNBgx6rizJMqQnf/X/Rt/unr+QISUqMqRERTZPnvFYy2crDkfMNxuy51M/pZ4vVZIUKZJHSBOBW1Yf3oCbN0mf7r9BOWvWrSdLFj/y5s71WJ9Txb0Tv27h9tVrEdIy5M/D8Y2u7phHVq+j5CsNAVf/+0RJfHF4eZHQxwfn3WDu3Aggba4cXPr9BDf//geAo2vWU/KVRo+3IJ7A4RXzzYY0aMezPj17MPLLcVSpU58RX4yjxztvAa7RjZOmzoh0WLvyDBcOHqFYw5cAKNmsMamy+gGw5/vFBN26zYiLxxl65hCrPx/H7WvXuHLiTzIWyEea7NlweHlRrHG98HMASrzSkH77ttDpuxmkyuIX6T0Vcdo8IiKnROSAiPwmIrustNQislpEjlv/pnLL31tETojIMRGp7ZZeyrrOCREZJ/LwbTMPHbRF5PUHHOskIrtEZJf/lGkPewuPMPe7H+j9wftsWLmc3j3fo+9A1wvEryb60+61lpFOIKU8w4w3ulHlrU703rWBxMmS4bzr6q+fs0wpTEgIH2fOR7+cRaj5wTukzZmD29evM6fr+7w5fxo9f/2Zf06dIdTpBGD/spX0zfEcnxWrwJE162k3/ZsnWbSnW9w3j1QzxhQ3xpS2PvcC1hpj8gJrrc+ISCGgBVAYqANMEJGw6vxEoBOQ19rqPGzxHqX3yEBgamQHjDH+gD/wzPfTXrT8R/p+9AEAdV+sSb9BQwHYd/AgP6/5hc+//JobAQE4HA4SJUzIay2aP8nHVXHo0rHjjKvdGID0efNQpJ6r4vV8q+YcWrmGUKeTgCt/88fmbWQvXYK/T57iwPKVHFi+EoCKHdtjrGleb129Gn7dTZOm0WRE9EPhn1nx/4KxEVDV2p8OrAc+ttLnGWOCgJMicgIoIyKngOTGmK2ux5MZQGNgxcPc/IFBW0T2R3UIyPAwN3zWpE+Xjh2791C2dCm27dhJDmvY8pwpk8LzfPWNP76+vhqwPUyydGkJuPI3IsJL/T5k4zeTAbh65iz5q1dm+6x5JPT1JVe55/nlywkRzvFNmZIq3d5kUvP2gGvBhRt/XQKgWMOXuHjk9ydSJluIRZc/EemEqwYcxt+qdIYxwCoRMcD/rGMZjDEXAYwxF0UkvZXXD9jmdu45Ky3Y2r83/aFEV9POANQGrt2TLsCWh72pp+rRqx87du/m2vXrVK5dn3e6dGTwJ30YOmoMTqeTRIkSMahf5NN2KnvrMGcK+apWJGnaNAw7e4RlA4aSOGlSqrzVEYC9C5eyZaqrW+eG8ZNoO3UC/Q9uR0TYMnUW5w+4uoI2HzuSLMVcs0H+OGgEl4+7hsBX796Fog1fItTp5NbVa0xv3+UJlNImYvGCMUKrQOReMMZcsALzahE5+oC8kVXxzQPSH8oDh7GLyGRgqjFmUyTH5hhjWkV7h2e8eURFToexq8jEyTD2jfNjPoy98qsxvp+IfArcBDoCVa1adiZgvTEmv4j0BjDGDLPy/wx8CpwC1hljCljpLa3zH2qimAf+HWGM6RBZwLaORR+wlVLqcRNHzLcHXUYkiYgkC9sHagEHgaVAOytbOyBsQdelQAsRSSQiOXG9cNxhNaUEiEg5q9dIW7dzYk2HsSulPEvcvYjMACyyeud5A3OMMStFZCewQEQ6AGeAZgDGmEMisgA4DDiBt4wxYQuGdgWmAT64XkA+1EvIsAdRSinPEUdzjxhj/gTum6DcGPMPUCOKc4YAQyJJ3wVEvXRVLOjgmoewcfNWajduyosNm+A/ZXqU+fYfOkzBUuVYuXpteNqNgAC69+xFnZebUbdJc/buc3XQGTX2Kxo0b8VH/QaE5128/Cemz5kXfwVRj0wcDvrs+ZVuyxYAUc83cq8hJw/wyf6t9N27KXz5MXcvfvAO35gbJEmTGoDcFcrSb98Weu1YTzprBK1PihS8s3JR/BTMxkQkxpsdadCOpZCQEAYNH8m3X4/lxx/ms3zlz5z4489I830+9isqli8XIX3IyNFUqlCOlYu+Y8n82eTOlZOAgJvs3befZQvmEBIayrHjJwgMDGTRsuW0atb0cRVNPYTq73blL7fud1HNNxKZMdXqMaRERYY9XzVCeqosfhR4sTr/nD4Tnlbzg3fwf6UNS/oMpHJX16IYL33yESuHfh63BfIEDu+YbzakQTuW9h88RPasWciaxY+ECRJQr3Yt1q7feF++mfMWULtGddKkDh/hys2bN9m5Zy9NX3bNJ5EwQQKSJ0uGOITgYCfGGIKCgvD29ubb6bNo0+JVEiSw5y/WsyClX2aK1KvN5m//+2srqvlGYqPZF8NY+NEn4NazKyQ4mAQ+iUno60NIcDBpc+UkpV/m8HspNx6+3JgG7Vi6dPkKGTP8N64oQ4b0XLpy5Z48l1nzy/r7FuQ9e/4CqVOloveAQTRu8Rp9B37G7Tt3SJokCbVqVKNxi9fIkjkzyZIm5eDhw9SsVgX19Gr+5XAWftQ/wirsUc03ci9jDO+uWkzvXRuo2LF9eHrRBnW5fv4i5/cfjJB/5bAxtPYfR/X3urH+a38aDenPsk90TvVIxVHvkaeVPZ/6CTKR9Im/9/t6yKgx9Hz3bby8InbydzqdHD56jJbNXmHxvFn4+PiEt4l3bN+WJfNn0+uD9xg74Ru6d+3MdwsX8+5HvZkwaXJ8FUc9pCL16hBw+W/O7PktQnpU843ca9QLtRhaqjJf132Fqm91JE+lCiTw8aFu3w9Z2v++91ic23eAkeVr8EX1+qTNlYN/L1wEEd6cN5XXZ04iWfp08VFMe/Lw+bT1b+9Yypg+PX9duhT++dKly6RPF/F/mIOHj9CjVz8Arl2/zoZNW/D29qJYkefImD49xYq4XiLXqVkd/6kR50U+fNS1qG+O7NkYMnI0s6f48/7HfTl1+gw5smeLz6KpWMj9QlmKNqzLcy+9iHfixPgkT8brMycxtU3HSOcbude/F/8CIODK3/y2aDk5y5Ti9rXrpMmZnU/2uZo8Umbxo++eXxlepho3Ll0OP/elfh/x7avtafH15ywbMJQ0ObJTvXsXlvQbHL+Ftgub1qBjSoN2LBUpXIhTZ85y9vx5MqRPz48/r2L0sIj/s/zy43/95nv1H0jVShWpWa0qABkzpufPU6fJlSM7W3fsJHeunBHOHTvhGwb164PT6STE+rPb4RACAwPjtVwqdhb3GcjiPq5Jm/JVqUjNnt2Z2qZjlPONuEvo64s4HATdvElCX18K1qrOj4NGcOHgYT7KkDs835CTBxhaugq3/vlvsqjy7Vpx8MefuX39Ogl9fTChBhMaSkKdLfI/Nq1Bx5QG7Vjy9vam/8cf8ma37oSEhvJKowbkzZ2bud/9AEDLZq888PxPPv6Qnn0+IdjpJKtfZoYN7B9+bM269RQpXIgM1p+6JYoWoUGzluTLm4cC+fPFX6FUnHm+ZbNI5xtJkSkjbb79mq/rNSV5hvR0WTQbcC09tnPOdxz+eU20107g40O5dq0YW6sxAGvGfE3nH2bivHuXyS07xE+B7MjLnosbxNQD5x6JEzr3iIqEzj2iIhMXc4+E7l8X45jjKFrNdtVyrWkrpTyLNo8opZSN6ItIpZSyEa1pK6WUjXh5dljz7NIppZ45dp0IKqY0aCulPIu2aSullI1oTVsppWxEa9pKKWUjHl7T9uyvJKXUs8fLK+bbA4hIVhFZJyJHROSQiLxrpX8qIudF5Ddre8ntnN4ickJEjolIbbf0UiJywDo2Th7hbanWtJVSniXumkecwAfGmD3Wquy7RWS1dewLY0yEZYNEpBDQAigMZAbWiEg+a3HfiUAnYBvwE1CHh1zcV2vaSinPEkfzaRtjLhpj9lj7AcARIPJVLVwaAfOMMUHGmJPACaCMiGQCkhtjthrXZE8zgMYPWzwN2kopDyMx3kSkk4jscts6RXpFkRxACWC7lfS2iOwXkSkiEramoB9w1u20c1aan7V/b/pD0aCtlPIssahpG2P8jTGl3Tb/+y8nSYEfgPeMMTdwNXXkBooDF4HRYVkjeRrzgPSHom3aSinPEoe9R0QkAa6APdsYsxDAGHPJ7fgkYLn18RzgPudwFuCClZ4lkvSHojVtpZRniaOFfa0eHpOBI8aYMW7pmdyyvQyErcK8FGghIolEJCeQF9hhjLkIBIhIOeuabYElPCStaSulPEvcVbRfANoAB0TkNyutD9BSRIrjauI4BXQGMMYcEpEFwGFcPU/esnqOAHQFpgE+uHqNPFTPEdCVa9QToivXqMjExco15sLvMY45kjmf7UbiaE1bKeVZPHxEpAZtpZRn0aCtlFI2ohNGKaWUnWhNWyml7EObR5RSykY0aCullJ1o0FZKKdvQhX2VUspOtPeIUkrZiNa0lVLKRjRoK6WUnWjQVkop+9CatlJK2Yhnx2wN2kopD6O9R5RSyka0eUQppexEg7ZSStmH1rSVUspGNGgrpZSNePiLyPhf2FeFE5FOxhj/J/0c6umivxcqNjz7K+np0+lJP4B6KunvhYoxDdpKKWUjGrSVUspGNGg/XtpuqSKjvxcqxvRFpFJK2YjWtJVSykY0aCullI1o0H5MRKSOiBwTkRMi0utJP4968kRkiohcFpGDT/pZlH1o0H4MRMQLGA/UBQoBLUWk0JN9KvUUmAbUedIPoexFg/bjUQY4YYz50xhzF5gHNHrCz6SeMGPMRuDqk34OZS8atB8PP+Cs2+dzVppSSsWKBu3HI7Jpx7SvpVIq1jRoPx7ngKxun7MAF57QsyilbEyD9uOxE8grIjlFJCHQAlj6hJ9JKWVDGrQfA2OME3gb+Bk4Aiwwxhx6sk+lnjQRmQtsBfKLyDkR6fCkn0k9/XQYu1JK2YjWtJVSykY0aCullI1o0FZKKRvRoK2UUjaiQVsppWxEg7ZSStmIBm2llLKR/wNZZQW09Qv5ygAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lsvc_matrix = confusion_matrix(y_test, nb_pred) \n",
    "group_names = ['True Neg', 'False Pos', 'False Neg', 'True Pos']\n",
    "group_counts = [\"{0:0.0f}\".format(value) for value in lsvc_matrix.flatten()]\n",
    "group_percentages = [\"{0:.2%}\".format(value) for value in lsvc_matrix.flatten()/np.sum(lsvc_matrix)]\n",
    "labels = [f\"{v1}\\n{v2}\\n{v3}\" for v1, v2, v3 in\n",
    "          zip(group_names,group_counts,group_percentages)]\n",
    "labels = np.asarray(labels).reshape(2,2)\n",
    "sns.heatmap(lsvc_matrix, annot=labels, fmt='', cmap='Reds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "d8677ea3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         0.0       0.52      0.97      0.68     20079\n",
      "         1.0       0.78      0.10      0.17     20079\n",
      "\n",
      "    accuracy                           0.53     40158\n",
      "   macro avg       0.65      0.53      0.43     40158\n",
      "weighted avg       0.65      0.53      0.43     40158\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#classification report\n",
    "lsvc_report = classification_report(y_test, lsvc_pred) \n",
    "print(lsvc_report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}