ML Algorithms for multiclass dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86372612",
   "metadata": {},
   "source": [
    "## Importing Libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "09d944be",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from statistics import mean\n",
    "from sklearn.metrics import accuracy_score,confusion_matrix,classification_report,precision_score,recall_score,f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16cfca4a",
   "metadata": {},
   "source": [
    "### Loading Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "750bc844",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>232.0</td>\n",
       "      <td>8153.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>420.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  src_bytes  dst_bytes  land  wrong_fragment  urgent  hot  \\\n",
       "0       0.0      491.0        0.0   0.0             0.0     0.0  0.0   \n",
       "1       0.0      146.0        0.0   0.0             0.0     0.0  0.0   \n",
       "2       0.0        0.0        0.0   0.0             0.0     0.0  0.0   \n",
       "3       0.0      232.0     8153.0   0.0             0.0     0.0  0.0   \n",
       "4       0.0      199.0      420.0   0.0             0.0     0.0  0.0   \n",
       "\n",
       "   num_failed_logins  logged_in  num_compromised  ...  \\\n",
       "0                0.0        0.0              0.0  ...   \n",
       "1                0.0        0.0              0.0  ...   \n",
       "2                0.0        0.0              0.0  ...   \n",
       "3                0.0        1.0              0.0  ...   \n",
       "4                0.0        1.0              0.0  ...   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.17                         0.00   \n",
       "1                         0.88                         0.00   \n",
       "2                         0.00                         0.00   \n",
       "3                         0.03                         0.04   \n",
       "4                         0.00                         0.00   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                  0.00                      0.00                  0.05   \n",
       "1                  0.00                      0.00                  0.00   \n",
       "2                  1.00                      1.00                  0.00   \n",
       "3                  0.03                      0.01                  0.00   \n",
       "4                  0.00                      0.00                  0.00   \n",
       "\n",
       "   dst_host_srv_rerror_rate  protocol_type  service  flag  label  \n",
       "0                      0.00            1.0     20.0   9.0    0.0  \n",
       "1                      0.00            2.0     44.0   9.0    0.0  \n",
       "2                      0.00            1.0     49.0   5.0    2.0  \n",
       "3                      0.01            1.0     24.0   9.0    0.0  \n",
       "4                      0.00            1.0     24.0   9.0    0.0  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('ids_multi.csv')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8580a617",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b3be2625",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((160367, 41), (160367,))"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = data.iloc[:,0:-1].values\n",
    "y = data.iloc[:,-1].values\n",
    "x.shape,y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2af71eba",
   "metadata": {},
   "source": [
    "### splitting the dataset into train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3f25f4ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7198d4ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((120275, 41), (40092, 41), (120275,), (40092,))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train,x_test,y_train,y_test = train_test_split(x,y)\n",
    "x_train.shape,x_test.shape,y_train.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c5fb1558",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_names = ['Normal','Probe','DOS','u2r','r2l']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99132e6e",
   "metadata": {},
   "source": [
    "# Applying Machine Learning Algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0719b42",
   "metadata": {},
   "source": [
    "## ADABoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b9dd8867",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "7adf3b14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier()"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79d49804",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "21f3281f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8599293286219081"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc_train_score = abc.score(x_train,y_train)\n",
    "abc_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "158bee06",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8a459eab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8568542352589045"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc_test_score = abc.score(x_test,y_test)\n",
    "abc_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5e1de25",
   "metadata": {},
   "source": [
    "### Predicted y values for train and test datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "851bd766",
   "metadata": {},
   "outputs": [],
   "source": [
    "abc_y_train = abc.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9a7d41b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "abc_y_test = abc.predict(x_test)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e1f8e12",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cc19494",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "22eae873",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[56103  1149  1021   908   101]\n",
      " [  892 10835   658     0     2]\n",
      " [ 6653   942 35648   121     0]\n",
      " [   52     1     1    81     7]\n",
      " [ 4032    42     1   264   761]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_train,abc_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e9a8960",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9696e848",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[18868   382   335   305    34]\n",
      " [  329  3512   248     0     3]\n",
      " [ 2287   340 11685    51     0]\n",
      " [   20     1     1    22     0]\n",
      " [ 1309    10     2    82   266]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,abc_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88bdb3bf",
   "metadata": {},
   "source": [
    "## Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b51eb69",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4644824d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      Normal       0.83      0.95      0.88     59282\n",
      "       Probe       0.84      0.87      0.85     12387\n",
      "         DOS       0.95      0.82      0.88     43364\n",
      "         u2r       0.06      0.57      0.11       142\n",
      "         r2l       0.87      0.15      0.25      5100\n",
      "\n",
      "    accuracy                           0.86    120275\n",
      "   macro avg       0.71      0.67      0.60    120275\n",
      "weighted avg       0.88      0.86      0.85    120275\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_train,abc_y_train, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88a15c8f",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9b1cdb1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      Normal       0.83      0.95      0.88     19924\n",
      "       Probe       0.83      0.86      0.84      4092\n",
      "         DOS       0.95      0.81      0.88     14363\n",
      "         u2r       0.05      0.50      0.09        44\n",
      "         r2l       0.88      0.16      0.27      1669\n",
      "\n",
      "    accuracy                           0.86     40092\n",
      "   macro avg       0.71      0.66      0.59     40092\n",
      "weighted avg       0.87      0.86      0.85     40092\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,abc_y_test, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90787335",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3d77f3d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training scores\n",
      "\n",
      "Precision score: 0.8599293286219081\n",
      "Recall score: 0.8599293286219081\n",
      "F1 score: 0.8599293286219082\n",
      "\n",
      "Testing scores\n",
      "\n",
      "Precision score: 0.8568542352589045\n",
      "Recall score: 0.8568542352589045\n",
      "F1 score: 0.8568542352589045\n"
     ]
    }
   ],
   "source": [
    "abc_precision_train = precision_score(y_train,abc_y_train,average='micro')\n",
    "abc_precision_test = precision_score(y_test,abc_y_test,average='micro')\n",
    "abc_recall_train = recall_score(y_train,abc_y_train,average='micro')\n",
    "abc_recall_test = recall_score(y_test,abc_y_test,average='micro')\n",
    "abc_f1_train = f1_score(y_train,abc_y_train,average='micro')\n",
    "abc_f1_test = f1_score(y_test,abc_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",abc_precision_train)\n",
    "print(\"Recall score:\",abc_recall_train)\n",
    "print(\"F1 score:\",abc_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",abc_precision_test)\n",
    "print(\"Recall score:\",abc_recall_test)\n",
    "print(\"F1 score:\",abc_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cef913e8",
   "metadata": {},
   "source": [
    "## K fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2d327fc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1a07f329",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Index:  [ 32074  32075  32076 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [    0     1     2 ... 32071 32072 32073]\n",
      "Train Index:  [     0      1      2 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [32074 32075 32076 ... 64145 64146 64147]\n",
      "Train Index:  [     0      1      2 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [64148 64149 64150 ... 96218 96219 96220]\n",
      "Train Index:  [     0      1      2 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [ 96221  96222  96223 ... 128291 128292 128293]\n",
      "Train Index:  [     0      1      2 ... 128291 128292 128293] \n",
      "\n",
      "Test Index:  [128294 128295 128296 ... 160364 160365 160366]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "abc_training_scores = []\n",
    "abc_testing_scores = []\n",
    "best_abc = AdaBoostClassifier()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_abc.fit(X_train, Y_train)\n",
    "    abc_training_scores.append(best_abc.score(X_train, Y_train))\n",
    "    abc_testing_scores.append(best_abc.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4e1a336d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8260856009291232,\n",
       " 0.8915373403069536,\n",
       " 0.8793786147442593,\n",
       " 0.8737197374779803,\n",
       " 0.9152883221350959]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8c364052",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.8487560017459624,\n",
       " 0.9219928914385483,\n",
       " 0.9138839522339662,\n",
       " 0.8972656128207527,\n",
       " 0.7217597356031553]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abc_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "667b8e27",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Accuracy for training data is :  0.8772019231186825\n",
      "Max Accuracy for training data is :  0.9152883221350959\n",
      "Average Accuracy for testing data is :  0.860731638768477\n",
      "Max Accuracy for testing data is :  0.9219928914385483\n"
     ]
    }
   ],
   "source": [
    "abc_avg_train = mean(abc_training_scores)\n",
    "abc_avg_test = mean(abc_testing_scores)\n",
    "abc_max_train = max(abc_training_scores)\n",
    "abc_max_test = max(abc_testing_scores)\n",
    "print(\"Average Accuracy for training data is : \",abc_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",abc_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",abc_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",abc_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43594c4c",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "89bfe317",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "xgbc = XGBClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "dd7324e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hebbar\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13:54:16] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "              colsample_bynode=1, colsample_bytree=1, enable_categorical=False,\n",
       "              gamma=0, gpu_id=-1, importance_type=None,\n",
       "              interaction_constraints='', learning_rate=0.300000012,\n",
       "              max_delta_step=0, max_depth=6, min_child_weight=1, missing=nan,\n",
       "              monotone_constraints='()', n_estimators=100, n_jobs=4,\n",
       "              num_parallel_tree=1, objective='multi:softprob', predictor='auto',\n",
       "              random_state=0, reg_alpha=0, reg_lambda=1, scale_pos_weight=None,\n",
       "              subsample=1, tree_method='exact', validate_parameters=1,\n",
       "              verbosity=None)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbc.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce464f1",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "1f232ac7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9993348576179588"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbc_train_score = xgbc.score(x_train,y_train)\n",
    "xgbc_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a83e0988",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "890c715f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9972563104858825"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgbc_test_score = xgbc.score(x_test,y_test)\n",
    "xgbc_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41e74e68",
   "metadata": {},
   "source": [
    "### Predicted Y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f76e72b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbc_y_train = xgbc.predict(x_train)\n",
    "xgbc_y_test = xgbc.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6984dbe",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34ea2322",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "769d0e67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[59245     1     4     0    32]\n",
      " [    0 12387     0     0     0]\n",
      " [    0     0 43364     0     0]\n",
      " [    1     0     0   141     0]\n",
      " [   42     0     0     0  5058]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_train,xgbc_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f870cb9d",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "55eba383",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[19885     7     6     2    24]\n",
      " [    9  4082     0     0     1]\n",
      " [    4     0 14359     0     0]\n",
      " [    3     0     0    39     2]\n",
      " [   51     0     0     1  1617]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(y_test,xgbc_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc82284d",
   "metadata": {},
   "source": [
    "### Classification Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98beea94",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9652b5f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      Normal       1.00      1.00      1.00     59282\n",
      "       Probe       1.00      1.00      1.00     12387\n",
      "         DOS       1.00      1.00      1.00     43364\n",
      "         u2r       1.00      0.99      1.00       142\n",
      "         r2l       0.99      0.99      0.99      5100\n",
      "\n",
      "    accuracy                           1.00    120275\n",
      "   macro avg       1.00      1.00      1.00    120275\n",
      "weighted avg       1.00      1.00      1.00    120275\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_train,xgbc_y_train, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "450a459f",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5259282d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      Normal       1.00      1.00      1.00     19924\n",
      "       Probe       1.00      1.00      1.00      4092\n",
      "         DOS       1.00      1.00      1.00     14363\n",
      "         u2r       0.93      0.89      0.91        44\n",
      "         r2l       0.98      0.97      0.98      1669\n",
      "\n",
      "    accuracy                           1.00     40092\n",
      "   macro avg       0.98      0.97      0.98     40092\n",
      "weighted avg       1.00      1.00      1.00     40092\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,xgbc_y_test, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d89291c6",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "cffb8052",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training scores\n",
      "\n",
      "Precision score: 0.9993348576179588\n",
      "Recall score: 0.9993348576179588\n",
      "F1 score: 0.9993348576179588\n",
      "\n",
      "Testing scores\n",
      "\n",
      "Precision score: 0.9972563104858825\n",
      "Recall score: 0.9972563104858825\n",
      "F1 score: 0.9972563104858825\n"
     ]
    }
   ],
   "source": [
    "xgbc_precision_train = precision_score(y_train,xgbc_y_train,average='micro')\n",
    "xgbc_precision_test = precision_score(y_test,xgbc_y_test,average='micro')\n",
    "xgbc_recall_train = recall_score(y_train,xgbc_y_train,average='micro')\n",
    "xgbc_recall_test = recall_score(y_test,xgbc_y_test,average='micro')\n",
    "xgbc_f1_train = f1_score(y_train,xgbc_y_train,average='micro')\n",
    "xgbc_f1_test = f1_score(y_test,xgbc_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",xgbc_precision_train)\n",
    "print(\"Recall score:\",xgbc_recall_train)\n",
    "print(\"F1 score:\",xgbc_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",xgbc_precision_test)\n",
    "print(\"Recall score:\",xgbc_recall_test)\n",
    "print(\"F1 score:\",xgbc_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f4c138",
   "metadata": {},
   "source": [
    "### K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2188816d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Index:  [ 32074  32075  32076 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [    0     1     2 ... 32071 32072 32073]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hebbar\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13:57:12] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Train Index:  [     0      1      2 ... 160364 160365 160366] \n",
      "\n",
      "Test Index:  [32074 32075 32076 ... 64145 64146 64147]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hebbar\\appdata\\local\\programs\\python\\python39\\lib\\site-packages\\xgboost\\sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[14:00:14] WARNING: C:/Users/Administrator/workspace/xgboost-win64_release_1.5.1/src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "xgbc_training_scores = []\n",
    "xgbc_testing_scores = []\n",
    "best_xgbc = XGBClassifier()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_xgbc.fit(X_train, Y_train)\n",
    "    xgbc_training_scores.append(best_xgbc.score(X_train, Y_train))\n",
    "    xgbc_testing_scores.append(best_xgbc.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d7759f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbc_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbd4800f",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbc_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "542c4a82",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbc_avg_train = mean(xgbc_training_scores)\n",
    "xgbc_avg_test = mean(xgbc_testing_scores)\n",
    "xgbc_max_train = max(xgbc_training_scores)\n",
    "xgbc_max_test = max(xgbc_testing_scores)\n",
    "print(\"Average Accuracy for training data is : \",xgbc_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",xgbc_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",xgbc_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",xgbc_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9787c1f7",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4326f27a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rfc = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "923c4b16",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0434a7d",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aa31c97",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_train_score = rfc.score(x_train,y_train)\n",
    "rfc_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb629809",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f644db4",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_test_score=rfc.score(x_test,y_test)\n",
    "rfc_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7fc898e",
   "metadata": {},
   "source": [
    "### Predicted y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "872c8237",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_y_train = rfc.predict(x_train)\n",
    "rfc_y_test = rfc.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c7edd9f",
   "metadata": {},
   "source": [
    "## Confusion matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87c95b35",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1de8282f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_train,rfc_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae4acc22",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a4d5f04",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_test,rfc_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7d27d4c",
   "metadata": {},
   "source": [
    "## Classification Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd447263",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b031d56",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_train,rfc_y_train, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eea31eda",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da8100e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test,rfc_y_test, target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ddf962c",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9785bb30",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_precision_train = precision_score(y_train,rfc_y_train,average='micro')\n",
    "rfc_precision_test = precision_score(y_test,rfc_y_test,average='micro')\n",
    "rfc_recall_train = recall_score(y_train,rfc_y_train,average='micro')\n",
    "rfc_recall_test = recall_score(y_test,rfc_y_test,average='micro')\n",
    "rfc_f1_train = f1_score(y_train,rfc_y_train,average='micro')\n",
    "rfc_f1_test = f1_score(y_test,rfc_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",rfc_precision_train)\n",
    "print(\"Recall score:\",rfc_recall_train)\n",
    "print(\"F1 score:\",rfc_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",rfc_precision_test)\n",
    "print(\"Recall score:\",rfc_recall_test)\n",
    "print(\"F1 score:\",rfc_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5369f8c9",
   "metadata": {},
   "source": [
    "### K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1f93280",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_training_scores = []\n",
    "rfc_testing_scores = []\n",
    "best_rfc = RandomForestClassifier()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_rfc.fit(X_train, Y_train)\n",
    "    rfc_training_scores.append(best_rfc.score(X_train, Y_train))\n",
    "    rfc_testing_scores.append(best_rfc.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcb7d88f",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75a43e15",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f28d8c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_avg_train = mean(rfc_training_scores)\n",
    "rfc_avg_test = mean(rfc_testing_scores)\n",
    "rfc_max_train = max(rfc_training_scores)\n",
    "rfc_max_test = max(rfc_testing_scores)\n",
    "print(\"Average Accuracy for training data is : \",rfc_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",rfc_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",rfc_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",rfc_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ca944c5",
   "metadata": {},
   "source": [
    "## Bernoulli Naive bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ec09655",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import BernoulliNB\n",
    "bnb = BernoulliNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "638b40e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6402804",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cfc07ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_train_score=bnb.score(x_train,y_train)\n",
    "bnb_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4d9980c",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4639915e",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_test_score=bnb.score(x_test,y_test)\n",
    "bnb_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8c5cfbe",
   "metadata": {},
   "source": [
    "### Predicting y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "621316c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_y_train = bnb.predict(x_train)\n",
    "bnb_y_test = bnb.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f6ec17d",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "386cca46",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cde7b4c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_train,bnb_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd4f2295",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dc25d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_test,bnb_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a585f880",
   "metadata": {},
   "source": [
    "## Classification Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "088ab076",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50af9f1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_train,bnb_y_train, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78c8e430",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "771e19a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test,bnb_y_test, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c73544f1",
   "metadata": {},
   "source": [
    "### Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9765ad34",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_precision_train = precision_score(y_train,bnb_y_train,average='micro')\n",
    "bnb_precision_test = precision_score(y_test,bnb_y_test,average='micro')\n",
    "bnb_recall_train = recall_score(y_train,bnb_y_train,average='micro')\n",
    "bnb_recall_test = recall_score(y_test,bnb_y_test,average='micro')\n",
    "bnb_f1_train = f1_score(y_train,bnb_y_train,average='micro')\n",
    "bnb_f1_test = f1_score(y_test,bnb_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",bnb_precision_train)\n",
    "print(\"Recall score:\",bnb_recall_train)\n",
    "print(\"F1 score:\",bnb_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",bnb_precision_test)\n",
    "print(\"Recall score:\",bnb_recall_test)\n",
    "print(\"F1 score:\",bnb_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f313487",
   "metadata": {},
   "source": [
    "## K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fa37c01",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_training_scores = []\n",
    "bnb_testing_scores = []\n",
    "best_bnb = BernoulliNB()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_bnb.fit(X_train, Y_train)\n",
    "    bnb_training_scores.append(best_bnb.score(X_train, Y_train))\n",
    "    bnb_testing_scores.append(best_bnb.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25a59f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff5a4257",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37665b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "bnb_avg_train = mean(bnb_training_scores)\n",
    "bnb_avg_test = mean(bnb_testing_scores)\n",
    "bnb_max_train = max(bnb_training_scores)\n",
    "bnb_max_test = max(bnb_testing_scores)\n",
    "print(\"Average Accuracy for training data is : \",bnb_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",bnb_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",bnb_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",bnb_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b2f49b3",
   "metadata": {},
   "source": [
    "## Support Vector Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47ae4566",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "svc = LinearSVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e5ec042",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4af4578",
   "metadata": {},
   "source": [
    "### Training Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e63a8ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_train_score=svc.score(x_train,y_train)\n",
    "svc_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "161f1ab2",
   "metadata": {},
   "source": [
    "### Testing Accurcy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "917964ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_test_score=svc.score(x_test,y_test)\n",
    "svc_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e054897",
   "metadata": {},
   "source": [
    "### Predicting y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d998d937",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_y_train = svc.predict(x_train)\n",
    "svc_y_test = svc.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83db8525",
   "metadata": {},
   "source": [
    "### Confusion Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "068c7806",
   "metadata": {},
   "source": [
    "### For training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cd6a194",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_train,svc_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1abe351c",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "543398af",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_test,svc_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c98d3ca3",
   "metadata": {},
   "source": [
    "### Classification Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d504fd1b",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "802b3bb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_train,svc_y_train, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "736a0572",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74f9aeef",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test,svc_y_test, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "94cc46cc",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c904fb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_precision_train = precision_score(y_train,svc_y_train,average='micro')\n",
    "svc_precision_test = precision_score(y_test,svc_y_test,average='micro')\n",
    "svc_recall_train = recall_score(y_train,svc_y_train,average='micro')\n",
    "svc_recall_test = recall_score(y_test,svc_y_test,average='micro')\n",
    "svc_f1_train = f1_score(y_train,svc_y_train,average='micro')\n",
    "svc_f1_test = f1_score(y_test,svc_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",svc_precision_train)\n",
    "print(\"Recall score:\",svc_recall_train)\n",
    "print(\"F1 score:\",svc_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",svc_precision_test)\n",
    "print(\"Recall score:\",svc_recall_test)\n",
    "print(\"F1 score:\",svc_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "930ffd63",
   "metadata": {},
   "source": [
    "### K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5a856c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_training_scores = []\n",
    "svc_testing_scores = []\n",
    "best_svc = LinearSVC()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_svc.fit(X_train, Y_train)\n",
    "    svc_training_scores.append(best_svc.score(X_train, Y_train))\n",
    "    svc_testing_scores.append(best_svc.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bc28a89",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13d6c35b",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a842aaf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "svc_avg_train = mean(svc_training_scores)\n",
    "svc_avg_test = mean(svc_testing_scores)\n",
    "svc_max_train = max(svc_training_scores)\n",
    "svc_max_test = max(svc_testing_scores)\n",
    "\n",
    "print(\"Average Accuracy for training data is : \",svc_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",svc_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",svc_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",svc_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b22e849c",
   "metadata": {},
   "source": [
    "## Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bae669db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "lr.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f4313b6",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc50fd35",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_train_score = lr.score(x_train,y_train)\n",
    "lr_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff8f6f2e",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0c18a5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_test_score = lr.score(x_test,y_test)\n",
    "lr_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5240ea6e",
   "metadata": {},
   "source": [
    "### Predicting y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afef0f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_y_train = lr.predict(x_train)\n",
    "lr_y_test = lr.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1d0205f",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5c18796",
   "metadata": {},
   "source": [
    "### For training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90b6233c",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_train,lr_y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3d35049",
   "metadata": {},
   "source": [
    "### For testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8666583",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(y_test,lr_y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7e8747c",
   "metadata": {},
   "source": [
    "## Classification report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "408211ca",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "472d1922",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_train,lr_y_train,target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12f9318f",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b050d1ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test,lr_y_test,target_names = target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a7f1954",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75d23c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_precision_train = precision_score(y_train,lr_y_train,average='micro')\n",
    "lr_precision_test = precision_score(y_test,lr_y_test,average='micro')\n",
    "lr_recall_train = recall_score(y_train,lr_y_train,average='micro')\n",
    "lr_recall_test = recall_score(y_test,lr_y_test,average='micro')\n",
    "lr_f1_train = f1_score(y_train,lr_y_train,average='micro')\n",
    "lr_f1_test = f1_score(y_test,lr_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",lr_precision_train)\n",
    "print(\"Recall score:\",lr_recall_train)\n",
    "print(\"F1 score:\",lr_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",lr_precision_test)\n",
    "print(\"Recall score:\",lr_recall_test)\n",
    "print(\"F1 score:\",lr_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea69053b",
   "metadata": {},
   "source": [
    "### K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f96d9b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_training_scores = []\n",
    "lr_testing_scores = []\n",
    "best_lr = LogisticRegression()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_lr.fit(X_train, Y_train)\n",
    "    lr_training_scores.append(best_lr.score(X_train, Y_train))\n",
    "    lr_testing_scores.append(best_lr.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e3cf8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c212bf9",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70ed99ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_avg_train = mean(lr_training_scores)\n",
    "lr_avg_test = mean(lr_testing_scores)\n",
    "lr_max_train = max(lr_training_scores)\n",
    "lr_max_test = max(lr_testing_scores)\n",
    "\n",
    "print(\"Average Accuracy for training data is : \",lr_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",lr_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",lr_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",lr_max_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cbc82280",
   "metadata": {},
   "source": [
    "## Decision Tree Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6deaa4a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dtc = DecisionTreeClassifier()\n",
    "dtc.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b909e1b9",
   "metadata": {},
   "source": [
    "### Training accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03d84074",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_train_score = dtc.score(x_train,y_train)\n",
    "dtc_train_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c0781d4",
   "metadata": {},
   "source": [
    "### Testing accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d093b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_test_score = dtc.score(x_test,y_test)\n",
    "dtc_test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb7aa5aa",
   "metadata": {},
   "source": [
    "### Predicting y values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe7a55bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_y_train = dtc.predict(x_train)\n",
    "dtc_y_test = dtc.predict(x_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b58a3bc1",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6d00788",
   "metadata": {},
   "source": [
    "### for training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50277852",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(dtc_y_train,y_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d39edf60",
   "metadata": {},
   "source": [
    "### for testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acf73bb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(dtc_y_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d4eaef",
   "metadata": {},
   "source": [
    "### Classification Report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5eb6553f",
   "metadata": {},
   "source": [
    "### For training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a91e317",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(dtc_y_train,y_train, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b185209",
   "metadata": {},
   "source": [
    "### For testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b408826d",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(dtc_y_test,y_test, target_names=target_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f765c823",
   "metadata": {},
   "source": [
    "### Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "421002cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_precision_train = precision_score(y_train,dtc_y_train,average='micro')\n",
    "dtc_precision_test = precision_score(y_test,dtc_y_test,average='micro')\n",
    "dtc_recall_train = recall_score(y_train,dtc_y_train,average='micro')\n",
    "dtc_recall_test = recall_score(y_test,dtc_y_test,average='micro')\n",
    "dtc_f1_train = f1_score(y_train,dtc_y_train,average='micro')\n",
    "dtc_f1_test = f1_score(y_test,dtc_y_test,average='micro')\n",
    "print(\"\\nTraining scores\\n\")\n",
    "print(\"Precision score:\",dtc_precision_train)\n",
    "print(\"Recall score:\",dtc_recall_train)\n",
    "print(\"F1 score:\",dtc_f1_train)\n",
    "print(\"\\nTesting scores\\n\")\n",
    "print(\"Precision score:\",dtc_precision_test)\n",
    "print(\"Recall score:\",dtc_recall_test)\n",
    "print(\"F1 score:\",dtc_f1_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c277104d",
   "metadata": {},
   "source": [
    "### K Fold Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1704eec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_training_scores = []\n",
    "dtc_testing_scores = []\n",
    "best_dtc = DecisionTreeClassifier()\n",
    "cv = KFold(n_splits=5, shuffle=False)\n",
    "\n",
    "for train_index, test_index in cv.split(x):\n",
    "    print(\"Train Index: \", train_index, \"\\n\")\n",
    "    print(\"Test Index: \", test_index)\n",
    "\n",
    "    X_train, X_test, Y_train, Y_test = x[train_index], x[test_index], y[train_index], y[test_index]\n",
    "    best_dtc.fit(X_train, Y_train)\n",
    "    dtc_training_scores.append(best_dtc.score(X_train, Y_train))\n",
    "    dtc_testing_scores.append(best_dtc.score(X_test, Y_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ca9ec79",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_training_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11d8662b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_testing_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fcaac70",
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_avg_train = mean(dtc_training_scores)\n",
    "dtc_avg_test = mean(dtc_testing_scores)\n",
    "dtc_max_train = max(dtc_training_scores)\n",
    "dtc_max_test = max(dtc_testing_scores)\n",
    "\n",
    "print(\"Average Accuracy for training data is : \",dtc_avg_train)\n",
    "print(\"Max Accuracy for training data is : \",dtc_max_train)\n",
    "print(\"Average Accuracy for testing data is : \",dtc_avg_test)\n",
    "print(\"Max Accuracy for testing data is : \",dtc_max_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f4628d9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd558313",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea1893c9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}