Decision Tree, Random Forest, AdaBoost, XGBoost & Hyper Parameter Optimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "bcacc0f9-8a6e-4999-9ff4-f3e412991f27",
   "metadata": {},
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b3018242-e354-4dca-9b95-0d5fc6162c7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split as tts, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import warnings \n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "76093bb0-5776-4dc1-8862-2873b716e9cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('uci breast cancer dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2be2790-2cc5-46a3-80fa-da32f8b87fb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>radius_mean</th>\n",
       "      <th>texture_mean</th>\n",
       "      <th>perimeter_mean</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>smoothness_mean</th>\n",
       "      <th>compactness_mean</th>\n",
       "      <th>concavity_mean</th>\n",
       "      <th>concave points_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>texture_worst</th>\n",
       "      <th>perimeter_worst</th>\n",
       "      <th>area_worst</th>\n",
       "      <th>smoothness_worst</th>\n",
       "      <th>compactness_worst</th>\n",
       "      <th>concavity_worst</th>\n",
       "      <th>concave points_worst</th>\n",
       "      <th>symmetry_worst</th>\n",
       "      <th>fractal_dimension_worst</th>\n",
       "      <th>Unnamed: 32</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>842302</td>\n",
       "      <td>M</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>...</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>842517</td>\n",
       "      <td>M</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>...</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>84300903</td>\n",
       "      <td>M</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>...</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>84348301</td>\n",
       "      <td>M</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>...</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>84358402</td>\n",
       "      <td>M</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>...</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id diagnosis  radius_mean  texture_mean  perimeter_mean  area_mean  \\\n",
       "0    842302         M        17.99         10.38          122.80     1001.0   \n",
       "1    842517         M        20.57         17.77          132.90     1326.0   \n",
       "2  84300903         M        19.69         21.25          130.00     1203.0   \n",
       "3  84348301         M        11.42         20.38           77.58      386.1   \n",
       "4  84358402         M        20.29         14.34          135.10     1297.0   \n",
       "\n",
       "   smoothness_mean  compactness_mean  concavity_mean  concave points_mean  \\\n",
       "0          0.11840           0.27760          0.3001              0.14710   \n",
       "1          0.08474           0.07864          0.0869              0.07017   \n",
       "2          0.10960           0.15990          0.1974              0.12790   \n",
       "3          0.14250           0.28390          0.2414              0.10520   \n",
       "4          0.10030           0.13280          0.1980              0.10430   \n",
       "\n",
       "   ...  texture_worst  perimeter_worst  area_worst  smoothness_worst  \\\n",
       "0  ...          17.33           184.60      2019.0            0.1622   \n",
       "1  ...          23.41           158.80      1956.0            0.1238   \n",
       "2  ...          25.53           152.50      1709.0            0.1444   \n",
       "3  ...          26.50            98.87       567.7            0.2098   \n",
       "4  ...          16.67           152.20      1575.0            0.1374   \n",
       "\n",
       "   compactness_worst  concavity_worst  concave points_worst  symmetry_worst  \\\n",
       "0             0.6656           0.7119                0.2654          0.4601   \n",
       "1             0.1866           0.2416                0.1860          0.2750   \n",
       "2             0.4245           0.4504                0.2430          0.3613   \n",
       "3             0.8663           0.6869                0.2575          0.6638   \n",
       "4             0.2050           0.4000                0.1625          0.2364   \n",
       "\n",
       "   fractal_dimension_worst  Unnamed: 32  \n",
       "0                  0.11890          NaN  \n",
       "1                  0.08902          NaN  \n",
       "2                  0.08758          NaN  \n",
       "3                  0.17300          NaN  \n",
       "4                  0.07678          NaN  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "613d535e-ab41-4050-8586-15da6740ffd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 569 entries, 0 to 568\n",
      "Data columns (total 33 columns):\n",
      " #   Column                   Non-Null Count  Dtype  \n",
      "---  ------                   --------------  -----  \n",
      " 0   id                       569 non-null    int64  \n",
      " 1   diagnosis                569 non-null    object \n",
      " 2   radius_mean              569 non-null    float64\n",
      " 3   texture_mean             569 non-null    float64\n",
      " 4   perimeter_mean           569 non-null    float64\n",
      " 5   area_mean                569 non-null    float64\n",
      " 6   smoothness_mean          569 non-null    float64\n",
      " 7   compactness_mean         569 non-null    float64\n",
      " 8   concavity_mean           569 non-null    float64\n",
      " 9   concave points_mean      569 non-null    float64\n",
      " 10  symmetry_mean            569 non-null    float64\n",
      " 11  fractal_dimension_mean   569 non-null    float64\n",
      " 12  radius_se                569 non-null    float64\n",
      " 13  texture_se               569 non-null    float64\n",
      " 14  perimeter_se             569 non-null    float64\n",
      " 15  area_se                  569 non-null    float64\n",
      " 16  smoothness_se            569 non-null    float64\n",
      " 17  compactness_se           569 non-null    float64\n",
      " 18  concavity_se             569 non-null    float64\n",
      " 19  concave points_se        569 non-null    float64\n",
      " 20  symmetry_se              569 non-null    float64\n",
      " 21  fractal_dimension_se     569 non-null    float64\n",
      " 22  radius_worst             569 non-null    float64\n",
      " 23  texture_worst            569 non-null    float64\n",
      " 24  perimeter_worst          569 non-null    float64\n",
      " 25  area_worst               569 non-null    float64\n",
      " 26  smoothness_worst         569 non-null    float64\n",
      " 27  compactness_worst        569 non-null    float64\n",
      " 28  concavity_worst          569 non-null    float64\n",
      " 29  concave points_worst     569 non-null    float64\n",
      " 30  symmetry_worst           569 non-null    float64\n",
      " 31  fractal_dimension_worst  569 non-null    float64\n",
      " 32  Unnamed: 32              0 non-null      float64\n",
      "dtypes: float64(31), int64(1), object(1)\n",
      "memory usage: 146.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "35d60447-5877-46fe-af33-f2e4ef1f6432",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(['id','Unnamed: 32'], axis = 'columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dd6bf424-9776-47e4-9bfe-2343215e66a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>radius_mean</th>\n",
       "      <th>texture_mean</th>\n",
       "      <th>perimeter_mean</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>smoothness_mean</th>\n",
       "      <th>compactness_mean</th>\n",
       "      <th>concavity_mean</th>\n",
       "      <th>concave points_mean</th>\n",
       "      <th>symmetry_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>radius_worst</th>\n",
       "      <th>texture_worst</th>\n",
       "      <th>perimeter_worst</th>\n",
       "      <th>area_worst</th>\n",
       "      <th>smoothness_worst</th>\n",
       "      <th>compactness_worst</th>\n",
       "      <th>concavity_worst</th>\n",
       "      <th>concave points_worst</th>\n",
       "      <th>symmetry_worst</th>\n",
       "      <th>fractal_dimension_worst</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M</td>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.30010</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>...</td>\n",
       "      <td>25.380</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.16220</td>\n",
       "      <td>0.66560</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M</td>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.08690</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>...</td>\n",
       "      <td>24.990</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.12380</td>\n",
       "      <td>0.18660</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M</td>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.19740</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>...</td>\n",
       "      <td>23.570</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.14440</td>\n",
       "      <td>0.42450</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M</td>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.24140</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>...</td>\n",
       "      <td>14.910</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.20980</td>\n",
       "      <td>0.86630</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M</td>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.19800</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>...</td>\n",
       "      <td>22.540</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.13740</td>\n",
       "      <td>0.20500</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>M</td>\n",
       "      <td>21.56</td>\n",
       "      <td>22.39</td>\n",
       "      <td>142.00</td>\n",
       "      <td>1479.0</td>\n",
       "      <td>0.11100</td>\n",
       "      <td>0.11590</td>\n",
       "      <td>0.24390</td>\n",
       "      <td>0.13890</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>...</td>\n",
       "      <td>25.450</td>\n",
       "      <td>26.40</td>\n",
       "      <td>166.10</td>\n",
       "      <td>2027.0</td>\n",
       "      <td>0.14100</td>\n",
       "      <td>0.21130</td>\n",
       "      <td>0.4107</td>\n",
       "      <td>0.2216</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.07115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>M</td>\n",
       "      <td>20.13</td>\n",
       "      <td>28.25</td>\n",
       "      <td>131.20</td>\n",
       "      <td>1261.0</td>\n",
       "      <td>0.09780</td>\n",
       "      <td>0.10340</td>\n",
       "      <td>0.14400</td>\n",
       "      <td>0.09791</td>\n",
       "      <td>0.1752</td>\n",
       "      <td>...</td>\n",
       "      <td>23.690</td>\n",
       "      <td>38.25</td>\n",
       "      <td>155.00</td>\n",
       "      <td>1731.0</td>\n",
       "      <td>0.11660</td>\n",
       "      <td>0.19220</td>\n",
       "      <td>0.3215</td>\n",
       "      <td>0.1628</td>\n",
       "      <td>0.2572</td>\n",
       "      <td>0.06637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>M</td>\n",
       "      <td>16.60</td>\n",
       "      <td>28.08</td>\n",
       "      <td>108.30</td>\n",
       "      <td>858.1</td>\n",
       "      <td>0.08455</td>\n",
       "      <td>0.10230</td>\n",
       "      <td>0.09251</td>\n",
       "      <td>0.05302</td>\n",
       "      <td>0.1590</td>\n",
       "      <td>...</td>\n",
       "      <td>18.980</td>\n",
       "      <td>34.12</td>\n",
       "      <td>126.70</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>0.11390</td>\n",
       "      <td>0.30940</td>\n",
       "      <td>0.3403</td>\n",
       "      <td>0.1418</td>\n",
       "      <td>0.2218</td>\n",
       "      <td>0.07820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>M</td>\n",
       "      <td>20.60</td>\n",
       "      <td>29.33</td>\n",
       "      <td>140.10</td>\n",
       "      <td>1265.0</td>\n",
       "      <td>0.11780</td>\n",
       "      <td>0.27700</td>\n",
       "      <td>0.35140</td>\n",
       "      <td>0.15200</td>\n",
       "      <td>0.2397</td>\n",
       "      <td>...</td>\n",
       "      <td>25.740</td>\n",
       "      <td>39.42</td>\n",
       "      <td>184.60</td>\n",
       "      <td>1821.0</td>\n",
       "      <td>0.16500</td>\n",
       "      <td>0.86810</td>\n",
       "      <td>0.9387</td>\n",
       "      <td>0.2650</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.12400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>B</td>\n",
       "      <td>7.76</td>\n",
       "      <td>24.54</td>\n",
       "      <td>47.92</td>\n",
       "      <td>181.0</td>\n",
       "      <td>0.05263</td>\n",
       "      <td>0.04362</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.1587</td>\n",
       "      <td>...</td>\n",
       "      <td>9.456</td>\n",
       "      <td>30.37</td>\n",
       "      <td>59.16</td>\n",
       "      <td>268.6</td>\n",
       "      <td>0.08996</td>\n",
       "      <td>0.06444</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.2871</td>\n",
       "      <td>0.07039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>569 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    diagnosis  radius_mean  texture_mean  perimeter_mean  area_mean  \\\n",
       "0           M        17.99         10.38          122.80     1001.0   \n",
       "1           M        20.57         17.77          132.90     1326.0   \n",
       "2           M        19.69         21.25          130.00     1203.0   \n",
       "3           M        11.42         20.38           77.58      386.1   \n",
       "4           M        20.29         14.34          135.10     1297.0   \n",
       "..        ...          ...           ...             ...        ...   \n",
       "564         M        21.56         22.39          142.00     1479.0   \n",
       "565         M        20.13         28.25          131.20     1261.0   \n",
       "566         M        16.60         28.08          108.30      858.1   \n",
       "567         M        20.60         29.33          140.10     1265.0   \n",
       "568         B         7.76         24.54           47.92      181.0   \n",
       "\n",
       "     smoothness_mean  compactness_mean  concavity_mean  concave points_mean  \\\n",
       "0            0.11840           0.27760         0.30010              0.14710   \n",
       "1            0.08474           0.07864         0.08690              0.07017   \n",
       "2            0.10960           0.15990         0.19740              0.12790   \n",
       "3            0.14250           0.28390         0.24140              0.10520   \n",
       "4            0.10030           0.13280         0.19800              0.10430   \n",
       "..               ...               ...             ...                  ...   \n",
       "564          0.11100           0.11590         0.24390              0.13890   \n",
       "565          0.09780           0.10340         0.14400              0.09791   \n",
       "566          0.08455           0.10230         0.09251              0.05302   \n",
       "567          0.11780           0.27700         0.35140              0.15200   \n",
       "568          0.05263           0.04362         0.00000              0.00000   \n",
       "\n",
       "     symmetry_mean  ...  radius_worst  texture_worst  perimeter_worst  \\\n",
       "0           0.2419  ...        25.380          17.33           184.60   \n",
       "1           0.1812  ...        24.990          23.41           158.80   \n",
       "2           0.2069  ...        23.570          25.53           152.50   \n",
       "3           0.2597  ...        14.910          26.50            98.87   \n",
       "4           0.1809  ...        22.540          16.67           152.20   \n",
       "..             ...  ...           ...            ...              ...   \n",
       "564         0.1726  ...        25.450          26.40           166.10   \n",
       "565         0.1752  ...        23.690          38.25           155.00   \n",
       "566         0.1590  ...        18.980          34.12           126.70   \n",
       "567         0.2397  ...        25.740          39.42           184.60   \n",
       "568         0.1587  ...         9.456          30.37            59.16   \n",
       "\n",
       "     area_worst  smoothness_worst  compactness_worst  concavity_worst  \\\n",
       "0        2019.0           0.16220            0.66560           0.7119   \n",
       "1        1956.0           0.12380            0.18660           0.2416   \n",
       "2        1709.0           0.14440            0.42450           0.4504   \n",
       "3         567.7           0.20980            0.86630           0.6869   \n",
       "4        1575.0           0.13740            0.20500           0.4000   \n",
       "..          ...               ...                ...              ...   \n",
       "564      2027.0           0.14100            0.21130           0.4107   \n",
       "565      1731.0           0.11660            0.19220           0.3215   \n",
       "566      1124.0           0.11390            0.30940           0.3403   \n",
       "567      1821.0           0.16500            0.86810           0.9387   \n",
       "568       268.6           0.08996            0.06444           0.0000   \n",
       "\n",
       "     concave points_worst  symmetry_worst  fractal_dimension_worst  \n",
       "0                  0.2654          0.4601                  0.11890  \n",
       "1                  0.1860          0.2750                  0.08902  \n",
       "2                  0.2430          0.3613                  0.08758  \n",
       "3                  0.2575          0.6638                  0.17300  \n",
       "4                  0.1625          0.2364                  0.07678  \n",
       "..                    ...             ...                      ...  \n",
       "564                0.2216          0.2060                  0.07115  \n",
       "565                0.1628          0.2572                  0.06637  \n",
       "566                0.1418          0.2218                  0.07820  \n",
       "567                0.2650          0.4087                  0.12400  \n",
       "568                0.0000          0.2871                  0.07039  \n",
       "\n",
       "[569 rows x 31 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "906bcae8-522d-455b-87fd-9725b19d99e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='diagnosis', ylabel='count'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAohElEQVR4nO3df3RU9Z3/8dckJEN+zWQTkkxSElRQIJIARQxzpBwKSAiB1TW2YhFQOXCggRbSApsuglA1igpUF2F1V8EtWSwVdE3LbyQoRMAoC/JrhUOFHjIJCybDjzIJZL5/9HC/nfJDCAkzfHg+zplzuD/mzvv2HMrTe+8kNr/f7xcAAIChwoI9AAAAQEsidgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgtFbBHiAUNDY26tixY4qLi5PNZgv2OAAA4Br4/X6dOnVKaWlpCgu78vUbYkfSsWPHlJ6eHuwxAABAExw9elRt27a94nZiR1JcXJykv/6P5XA4gjwNAAC4Fl6vV+np6da/41dC7EjWrSuHw0HsAABwi/muR1B4QBkAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNFaBXsAALjVHZmdFewRgJCUMWN3sEeQxJUdAABgOGIHAAAYLaixs3DhQmVnZ8vhcMjhcMjtdmvVqlXW9r59+8pmswW8xo0bF3CMI0eOKD8/X9HR0UpOTtaUKVN0/vz5m30qAAAgRAX1mZ22bdvqxRdf1N133y2/368lS5booYce0pdffql7771XkjRmzBjNnj3bek90dLT15wsXLig/P18ul0tbt25VVVWVRo4cqYiICL3wwgs3/XwAAEDoCWrsDB06NGD5+eef18KFC/XZZ59ZsRMdHS2Xy3XZ969du1Z79+7V+vXrlZKSom7duunXv/61pk2bpmeffVaRkZEtfg4AACC0hcwzOxcuXNCyZct05swZud1ua/3SpUvVpk0bdenSRcXFxTp79qy1raKiQllZWUpJSbHW5ebmyuv1as+ePVf8LJ/PJ6/XG/ACAABmCvpXz3fv3i23261z584pNjZWK1euVGZmpiTpJz/5idq1a6e0tDTt2rVL06ZN04EDB7RixQpJksfjCQgdSdayx+O54meWlJRo1qxZLXRGAAAglAQ9djp27KidO3eqrq5Ov//97zVq1CiVl5crMzNTY8eOtfbLyspSamqq+vfvr0OHDql9+/ZN/szi4mIVFRVZy16vV+np6Td0HgAAIDQF/TZWZGSkOnTooB49eqikpERdu3bVb37zm8vum5OTI0k6ePCgJMnlcqm6ujpgn4vLV3rOR5Lsdrv1DbCLLwAAYKagx87fa2xslM/nu+y2nTt3SpJSU1MlSW63W7t371ZNTY21z7p16+RwOKxbYQAA4PYW1NtYxcXFysvLU0ZGhk6dOqXS0lJt2rRJa9as0aFDh1RaWqrBgwcrMTFRu3bt0uTJk9WnTx9lZ2dLkgYOHKjMzEyNGDFCc+bMkcfj0fTp01VYWCi73R7MUwMAACEiqLFTU1OjkSNHqqqqSk6nU9nZ2VqzZo0efPBBHT16VOvXr9f8+fN15swZpaenq6CgQNOnT7feHx4errKyMo0fP15ut1sxMTEaNWpUwM/lAQAAtzeb3+/3B3uIYPN6vXI6naqrq+P5HQDXjV8EClxeS/8i0Gv99zvkntkBAABoTsQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwWlBjZ+HChcrOzpbD4ZDD4ZDb7daqVaus7efOnVNhYaESExMVGxurgoICVVdXBxzjyJEjys/PV3R0tJKTkzVlyhSdP3/+Zp8KAAAIUUGNnbZt2+rFF19UZWWlPv/8c/Xr108PPfSQ9uzZI0maPHmyPvroIy1fvlzl5eU6duyYHnnkEev9Fy5cUH5+vurr67V161YtWbJEixcv1owZM4J1SgAAIMTY/H6/P9hD/K2EhAS9/PLLevTRR5WUlKTS0lI9+uijkqT9+/erc+fOqqioUK9evbRq1SoNGTJEx44dU0pKiiRp0aJFmjZtmo4fP67IyMhr+kyv1yun06m6ujo5HI4WOzcAZjoyOyvYIwAhKWPG7hY9/rX++x0yz+xcuHBBy5Yt05kzZ+R2u1VZWamGhgYNGDDA2qdTp07KyMhQRUWFJKmiokJZWVlW6EhSbm6uvF6vdXXocnw+n7xeb8ALAACYKeixs3v3bsXGxsput2vcuHFauXKlMjMz5fF4FBkZqfj4+ID9U1JS5PF4JEkejycgdC5uv7jtSkpKSuR0Oq1Xenp6854UAAAIGUGPnY4dO2rnzp3atm2bxo8fr1GjRmnv3r0t+pnFxcWqq6uzXkePHm3RzwMAAMHTKtgDREZGqkOHDpKkHj16aMeOHfrNb36jxx57TPX19aqtrQ24ulNdXS2XyyVJcrlc2r59e8DxLn5b6+I+l2O322W325v5TAAAQCgK+pWdv9fY2Cifz6cePXooIiJCGzZssLYdOHBAR44ckdvtliS53W7t3r1bNTU11j7r1q2Tw+FQZmbmTZ8dAACEnqBe2SkuLlZeXp4yMjJ06tQplZaWatOmTVqzZo2cTqdGjx6toqIiJSQkyOFwaOLEiXK73erVq5ckaeDAgcrMzNSIESM0Z84ceTweTZ8+XYWFhVy5AQAAkoIcOzU1NRo5cqSqqqrkdDqVnZ2tNWvW6MEHH5QkzZs3T2FhYSooKJDP51Nubq7eeOMN6/3h4eEqKyvT+PHj5Xa7FRMTo1GjRmn27NnBOiUAABBiQu7n7AQDP2cHwI3g5+wAl8fP2QEAALgJiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABgtqLFTUlKinj17Ki4uTsnJyXr44Yd14MCBgH369u0rm80W8Bo3blzAPkeOHFF+fr6io6OVnJysKVOm6Pz58zfzVAAAQIhqFcwPLy8vV2FhoXr27Knz58/rV7/6lQYOHKi9e/cqJibG2m/MmDGaPXu2tRwdHW39+cKFC8rPz5fL5dLWrVtVVVWlkSNHKiIiQi+88MJNPR8AABB6gho7q1evDlhevHixkpOTVVlZqT59+ljro6Oj5XK5LnuMtWvXau/evVq/fr1SUlLUrVs3/frXv9a0adP07LPPKjIy8pL3+Hw++Xw+a9nr9TbTGQEAgFATUs/s1NXVSZISEhIC1i9dulRt2rRRly5dVFxcrLNnz1rbKioqlJWVpZSUFGtdbm6uvF6v9uzZc9nPKSkpkdPptF7p6ektcDYAACAUBPXKzt9qbGzUpEmT9MADD6hLly7W+p/85Cdq166d0tLStGvXLk2bNk0HDhzQihUrJEkejycgdCRZyx6P57KfVVxcrKKiImvZ6/USPAAAGCpkYqewsFBfffWVPv3004D1Y8eOtf6clZWl1NRU9e/fX4cOHVL79u2b9Fl2u112u/2G5gUAALeGkLiNNWHCBJWVlenjjz9W27Ztr7pvTk6OJOngwYOSJJfLperq6oB9Li5f6TkfAABw+whq7Pj9fk2YMEErV67Uxo0bdeedd37ne3bu3ClJSk1NlSS53W7t3r1bNTU11j7r1q2Tw+FQZmZmi8wNAABuHUG9jVVYWKjS0lJ9+OGHiouLs56xcTqdioqK0qFDh1RaWqrBgwcrMTFRu3bt0uTJk9WnTx9lZ2dLkgYOHKjMzEyNGDFCc+bMkcfj0fTp01VYWMitKgAAENwrOwsXLlRdXZ369u2r1NRU6/Xee+9JkiIjI7V+/XoNHDhQnTp10i9+8QsVFBToo48+so4RHh6usrIyhYeHy+1264knntDIkSMDfi4PAAC4fQX1yo7f77/q9vT0dJWXl3/ncdq1a6c//vGPzTUWAAAwSEg8oAwAANBSiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYLRWwR7gdtFjyrvBHgEISZUvjwz2CAAMx5UdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABitSbHTr18/1dbWXrLe6/WqX79+NzoTAABAs2lS7GzatEn19fWXrD937pw++eSTaz5OSUmJevbsqbi4OCUnJ+vhhx/WgQMHLjlmYWGhEhMTFRsbq4KCAlVXVwfsc+TIEeXn5ys6OlrJycmaMmWKzp8/35RTAwAAhrmuHyq4a9cu68979+6Vx+Oxli9cuKDVq1fre9/73jUfr7y8XIWFherZs6fOnz+vX/3qVxo4cKD27t2rmJgYSdLkyZP1hz/8QcuXL5fT6dSECRP0yCOPaMuWLdbn5ufny+VyaevWraqqqtLIkSMVERGhF1544XpODwAAGMjm9/v917pzWFiYbDabJOlyb4uKitLrr7+up59+uknDHD9+XMnJySovL1efPn1UV1enpKQklZaW6tFHH5Uk7d+/X507d1ZFRYV69eqlVatWaciQITp27JhSUlIkSYsWLdK0adN0/PhxRUZGfufner1eOZ1O1dXVyeFwNGn278JPUAYuz4SfoHxkdlawRwBCUsaM3S16/Gv99/u6ruwcPnxYfr9fd911l7Zv366kpCRrW2RkpJKTkxUeHt7koevq6iRJCQkJkqTKyko1NDRowIAB1j6dOnVSRkaGFTsVFRXKysqyQkeScnNzNX78eO3Zs0fdu3e/5HN8Pp98Pp+17PV6mzwzAAAIbdcVO+3atZMkNTY2NvsgjY2NmjRpkh544AF16dJFkuTxeBQZGan4+PiAfVNSUqxbaB6PJyB0Lm6/uO1ySkpKNGvWrGY+AwAAEIqa/ItAv/76a3388ceqqam5JH5mzJhx3ccrLCzUV199pU8//bSpI12z4uJiFRUVWcter1fp6ekt/rkAAODma1LsvPXWWxo/frzatGkjl8tlPccjSTab7bpjZ8KECSorK9PmzZvVtm1ba73L5VJ9fb1qa2sDru5UV1fL5XJZ+2zfvj3geBe/rXVxn79nt9tlt9uva0YAAHBratJXz5977jk9//zz8ng82rlzp7788kvr9cUXX1zzcfx+vyZMmKCVK1dq48aNuvPOOwO29+jRQxEREdqwYYO17sCBAzpy5Ijcbrckye12a/fu3aqpqbH2WbdunRwOhzIzM5tyegAAwCBNurLz7bff6kc/+tENf3hhYaFKS0v14YcfKi4uznrGxul0KioqSk6nU6NHj1ZRUZESEhLkcDg0ceJEud1u9erVS5I0cOBAZWZmasSIEZozZ448Ho+mT5+uwsJCrt4AAICmXdn50Y9+pLVr197why9cuFB1dXXq27evUlNTrdd7771n7TNv3jwNGTJEBQUF6tOnj1wul1asWGFtDw8PV1lZmcLDw+V2u/XEE09o5MiRmj179g3PBwAAbn1NurLToUMHPfPMM/rss8+UlZWliIiIgO0/+9nPruk41/Ijflq3bq0FCxZowYIFV9ynXbt2+uMf/3hNnwkAAG4vTYqdN998U7GxsSovL1d5eXnANpvNds2xAwAA0NKaFDuHDx9u7jkAAABaRJOe2QEAALhVNOnKznf97qu33367ScMAAAA0tyZ/9fxvNTQ06KuvvlJtba369evXLIMBAAA0hybFzsqVKy9Z19jYqPHjx6t9+/Y3PBQAAEBzabZndsLCwlRUVKR58+Y11yEBAABuWLM+oHzo0CGdP3++OQ8JAABwQ5p0G+tvf2O49NcfDlhVVaU//OEPGjVqVLMMBgAA0ByaFDtffvllwHJYWJiSkpL06quvfuc3tQAAAG6mJsXOxx9/3NxzAAAAtIgmxc5Fx48f14EDByRJHTt2VFJSUrMMBQAA0Fya9IDymTNn9PTTTys1NVV9+vRRnz59lJaWptGjR+vs2bPNPSMAAECTNSl2ioqKVF5ero8++ki1tbWqra3Vhx9+qPLycv3iF79o7hkBAACarEm3sd5//339/ve/V9++fa11gwcPVlRUlH784x9r4cKFzTUfAADADWnSlZ2zZ88qJSXlkvXJycncxgIAACGlSbHjdrs1c+ZMnTt3zlr3l7/8RbNmzZLb7W624QAAAG5Uk25jzZ8/X4MGDVLbtm3VtWtXSdL//M//yG63a+3atc06IAAAwI1oUuxkZWXp66+/1tKlS7V//35J0uOPP67hw4crKiqqWQcEAAC4EU2KnZKSEqWkpGjMmDEB699++20dP35c06ZNa5bhAAAAblSTntn5t3/7N3Xq1OmS9ffee68WLVp0w0MBAAA0lybFjsfjUWpq6iXrk5KSVFVVdcNDAQAANJcmxU56erq2bNlyyfotW7YoLS3thocCAABoLk16ZmfMmDGaNGmSGhoa1K9fP0nShg0bNHXqVH6CMgAACClNip0pU6boxIkT+ulPf6r6+npJUuvWrTVt2jQVFxc364AAAAA3okmxY7PZ9NJLL+mZZ57Rvn37FBUVpbvvvlt2u7255wMAALghTYqdi2JjY9WzZ8/mmgUAAKDZNekBZQAAgFsFsQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjBTV2Nm/erKFDhyotLU02m00ffPBBwPYnn3xSNpst4DVo0KCAfU6ePKnhw4fL4XAoPj5eo0eP1unTp2/iWQAAgFAW1Ng5c+aMunbtqgULFlxxn0GDBqmqqsp6/dd//VfA9uHDh2vPnj1at26dysrKtHnzZo0dO7alRwcAALeIG/qt5zcqLy9PeXl5V93HbrfL5XJddtu+ffu0evVq7dixQ/fdd58k6fXXX9fgwYP1yiuvKC0trdlnBgAAt5aQf2Zn06ZNSk5OVseOHTV+/HidOHHC2lZRUaH4+HgrdCRpwIABCgsL07Zt2654TJ/PJ6/XG/ACAABmCunYGTRokN59911t2LBBL730ksrLy5WXl6cLFy5Ikjwej5KTkwPe06pVKyUkJMjj8VzxuCUlJXI6ndYrPT29Rc8DAAAET1BvY32XYcOGWX/OyspSdna22rdvr02bNql///5NPm5xcbGKioqsZa/XS/AAAGCokL6y8/fuuusutWnTRgcPHpQkuVwu1dTUBOxz/vx5nTx58orP+Uh/fQ7I4XAEvAAAgJluqdj585//rBMnTig1NVWS5Ha7VVtbq8rKSmufjRs3qrGxUTk5OcEaEwAAhJCg3sY6ffq0dZVGkg4fPqydO3cqISFBCQkJmjVrlgoKCuRyuXTo0CFNnTpVHTp0UG5uriSpc+fOGjRokMaMGaNFixapoaFBEyZM0LBhw/gmFgAAkBTkKzuff/65unfvru7du0uSioqK1L17d82YMUPh4eHatWuX/vEf/1H33HOPRo8erR49euiTTz6R3W63jrF06VJ16tRJ/fv31+DBg9W7d2+9+eabwTolAAAQYoJ6Zadv377y+/1X3L5mzZrvPEZCQoJKS0ubcywAAGCQW+qZHQAAgOtF7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGhBjZ3Nmzdr6NChSktLk81m0wcffBCw3e/3a8aMGUpNTVVUVJQGDBigr7/+OmCfkydPavjw4XI4HIqPj9fo0aN1+vTpm3gWAAAglAU1ds6cOaOuXbtqwYIFl90+Z84cvfbaa1q0aJG2bdummJgY5ebm6ty5c9Y+w4cP1549e7Ru3TqVlZVp8+bNGjt27M06BQAAEOJaBfPD8/LylJeXd9ltfr9f8+fP1/Tp0/XQQw9Jkt59912lpKTogw8+0LBhw7Rv3z6tXr1aO3bs0H333SdJev311zV48GC98sorSktLu+yxfT6ffD6ftez1epv5zAAAQKgI2Wd2Dh8+LI/HowEDBljrnE6ncnJyVFFRIUmqqKhQfHy8FTqSNGDAAIWFhWnbtm1XPHZJSYmcTqf1Sk9Pb7kTAQAAQRWysePxeCRJKSkpAetTUlKsbR6PR8nJyQHbW7VqpYSEBGufyykuLlZdXZ31Onr0aDNPDwAAQkVQb2MFi91ul91uD/YYAADgJgjZKzsul0uSVF1dHbC+urra2uZyuVRTUxOw/fz58zp58qS1DwAAuL2FbOzceeedcrlc2rBhg7XO6/Vq27ZtcrvdkiS3263a2lpVVlZa+2zcuFGNjY3Kycm56TMDAIDQE9TbWKdPn9bBgwet5cOHD2vnzp1KSEhQRkaGJk2apOeee05333237rzzTj3zzDNKS0vTww8/LEnq3LmzBg0apDFjxmjRokVqaGjQhAkTNGzYsCt+EwsAANxegho7n3/+uX74wx9ay0VFRZKkUaNGafHixZo6darOnDmjsWPHqra2Vr1799bq1avVunVr6z1Lly7VhAkT1L9/f4WFhamgoECvvfbaTT8XAAAQmmx+v98f7CGCzev1yul0qq6uTg6Ho0U+o8eUd1vkuMCtrvLlkcEe4YYdmZ0V7BGAkJQxY3eLHv9a//0O2Wd2AAAAmgOxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADAasQMAAIxG7AAAAKMROwAAwGjEDgAAMBqxAwAAjBbSsfPss8/KZrMFvDp16mRtP3funAoLC5WYmKjY2FgVFBSouro6iBMDAIBQE9KxI0n33nuvqqqqrNenn35qbZs8ebI++ugjLV++XOXl5Tp27JgeeeSRIE4LAABCTatgD/BdWrVqJZfLdcn6uro6/cd//IdKS0vVr18/SdI777yjzp0767PPPlOvXr2ueEyfzyefz2cte73e5h8cAACEhJC/svP1118rLS1Nd911l4YPH64jR45IkiorK9XQ0KABAwZY+3bq1EkZGRmqqKi46jFLSkrkdDqtV3p6eoueAwAACJ6Qjp2cnBwtXrxYq1ev1sKFC3X48GH94Ac/0KlTp+TxeBQZGan4+PiA96SkpMjj8Vz1uMXFxaqrq7NeR48ebcGzAAAAwRTSt7Hy8vKsP2dnZysnJ0ft2rXT7373O0VFRTX5uHa7XXa7vTlGBAAAIS6kr+z8vfj4eN1zzz06ePCgXC6X6uvrVVtbG7BPdXX1ZZ/xAQAAt6dbKnZOnz6tQ4cOKTU1VT169FBERIQ2bNhgbT9w4ICOHDkit9sdxCkBAEAoCenbWL/85S81dOhQtWvXTseOHdPMmTMVHh6uxx9/XE6nU6NHj1ZRUZESEhLkcDg0ceJEud3uq34TCwAA3F5COnb+/Oc/6/HHH9eJEyeUlJSk3r1767PPPlNSUpIkad68eQoLC1NBQYF8Pp9yc3P1xhtvBHlqAAAQSkI6dpYtW3bV7a1bt9aCBQu0YMGCmzQRAAC41dxSz+wAAABcL2IHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQAAYDRiBwAAGI3YAQAARiN2AACA0YgdAABgNGIHAAAYzZjYWbBgge644w61bt1aOTk52r59e7BHAgAAIcCI2HnvvfdUVFSkmTNn6osvvlDXrl2Vm5urmpqaYI8GAACCzIjYmTt3rsaMGaOnnnpKmZmZWrRokaKjo/X2228HezQAABBkrYI9wI2qr69XZWWliouLrXVhYWEaMGCAKioqLvsen88nn89nLdfV1UmSvF5vi815wfeXFjs2cCtryb93N8upcxeCPQIQklr67/fF4/v9/qvud8vHzv/93//pwoULSklJCVifkpKi/fv3X/Y9JSUlmjVr1iXr09PTW2RGAFfmfH1csEcA0FJKnDflY06dOiWn88qfdcvHTlMUFxerqKjIWm5sbNTJkyeVmJgom80WxMlwM3i9XqWnp+vo0aNyOBzBHgdAM+Lv9+3F7/fr1KlTSktLu+p+t3zstGnTRuHh4aqurg5YX11dLZfLddn32O122e32gHXx8fEtNSJClMPh4P8MAUPx9/v2cbUrOhfd8g8oR0ZGqkePHtqwYYO1rrGxURs2bJDb7Q7iZAAAIBTc8ld2JKmoqEijRo3Sfffdp/vvv1/z58/XmTNn9NRTTwV7NAAAEGRGxM5jjz2m48ePa8aMGfJ4POrWrZtWr159yUPLgPTX25gzZ8685FYmgFsff79xOTb/d31fCwAA4BZ2yz+zAwAAcDXEDgAAMBqxAwAAjEbsAAAAoxE7MN6TTz4pm82mceMu/bUEhYWFstlsevLJJ2/+YACaxcW/4xdfiYmJGjRokHbt2hXs0RAiiB3cFtLT07Vs2TL95S///xeynjt3TqWlpcrIyAjiZACaw6BBg1RVVaWqqipt2LBBrVq10pAhQ4I9FkIEsYPbwve//32lp6drxYoV1roVK1YoIyND3bt3D+JkAJqD3W6Xy+WSy+VSt27d9M///M86evSojh8/HuzREAKIHdw2nn76ab3zzjvW8ttvv81P2QYMdPr0af32t79Vhw4dlJiYGOxxEAKIHdw2nnjiCX366af65ptv9M0332jLli164okngj0WgGZQVlam2NhYxcbGKi4uTv/93/+t9957T2Fh/DMHQ35dBHAtkpKSlJ+fr8WLF8vv9ys/P19t2rQJ9lgAmsEPf/hDLVy4UJL07bff6o033lBeXp62b9+udu3aBXk6BBuxg9vK008/rQkTJkiSFixYEORpADSXmJgYdejQwVr+93//dzmdTr311lt67rnngjgZQgGxg9vKoEGDVF9fL5vNptzc3GCPA6CF2Gw2hYWFBXwDE7cvYge3lfDwcO3bt8/6MwAz+Hw+eTweSX+9jfWv//qvOn36tIYOHRrkyRAKiB3cdhwOR7BHANDMVq9erdTUVElSXFycOnXqpOXLl6tv377BHQwhweb3+/3BHgIAAKCl8J08AABgNGIHAAAYjdgBAABGI3YAAIDRiB0AAGA0YgcAABiN2AEAAEYjdgAAgNGIHQBB07dvX02aNEmSdMcdd2j+/PlBned6/elPf5LNZtPOnTuDPQqAq+DXRQAICTt27FBMTEywx7gu6enpqqqqUps2bYI9CoCrIHYAhISkpKRgj3DdwsPD5XK5gj0GgO/AbSwAN8WZM2c0cuRIxcbGKjU1Va+++mrA9r+/jTV37lxlZWUpJiZG6enp+ulPf6rTp08HvOett95Senq6oqOj9U//9E+aO3eu4uPjre3PPvusunXrpv/8z//UHXfcIafTqWHDhunUqVPWPj6fTz/72c+UnJys1q1bq3fv3tqxY4e1/dtvv9Xw4cOVlJSkqKgo3X333XrnnXckXXob62r7AggeYgfATTFlyhSVl5frww8/1Nq1a7Vp0yZ98cUXV9w/LCxMr732mvbs2aMlS5Zo48aNmjp1qrV9y5YtGjdunH7+859r586devDBB/X8889fcpxDhw7pgw8+UFlZmcrKylReXq4XX3zR2j516lS9//77WrJkib744gt16NBBubm5OnnypCTpmWee0d69e7Vq1Srt27dPCxcuvOJtq+vZF8BN5AeAFnbq1Cl/ZGSk/3e/+5217sSJE/6oqCj/z3/+c7/f7/e3a9fOP2/evCseY/ny5f7ExERr+bHHHvPn5+cH7DN8+HC/0+m0lmfOnOmPjo72e71ea92UKVP8OTk5fr/f7z99+rQ/IiLCv3TpUmt7fX29Py0tzT9nzhy/3+/3Dx061P/UU09ddqbDhw/7Jfm//PLL79wXQPBwZQdAizt06JDq6+uVk5NjrUtISFDHjh2v+J7169erf//++t73vqe4uDiNGDFCJ06c0NmzZyVJBw4c0P333x/wnr9flv56eywuLs5aTk1NVU1NjTVXQ0ODHnjgAWt7RESE7r//fu3bt0+SNH78eC1btkzdunXT1KlTtXXr1ivOfD37Arh5iB0AIedPf/qThgwZouzsbL3//vuqrKzUggULJEn19fXXdayIiIiAZZvNpsbGxmt+f15enr755htNnjxZx44dU//+/fXLX/7yhvcFcPMQOwBaXPv27RUREaFt27ZZ67799lv97//+72X3r6ysVGNjo1599VX16tVL99xzj44dOxawT8eOHQMeJJZ0yfK1zBUZGaktW7ZY6xoaGrRjxw5lZmZa65KSkjRq1Cj99re/1fz58/Xmm29e8ZjXsy+Am4OvngNocbGxsRo9erSmTJmixMREJScn61/+5V8UFnb5/97q0KGDGhoa9Prrr2vo0KHasmWLFi1aFLDPxIkT1adPH82dO1dDhw7Vxo0btWrVKtlstmueKyYmRuPHj9eUKVOUkJCgjIwMzZkzR2fPntXo0aMlSTNmzFCPHj107733yufzqaysTJ07d77s8a5nXwA3D1d2ANwUL7/8sn7wgx9o6NChGjBggHr37q0ePXpcdt+uXbtq7ty5eumll9SlSxctXbpUJSUlAfs88MADWrRokebOnauuXbtq9erVmjx5slq3bn1dc7344osqKCjQiBEj9P3vf18HDx7UmjVr9A//8A+SpMjISBUXFys7O1t9+vRReHi4li1bdtljXc++AG4em9/v9wd7CABoDmPGjNH+/fv1ySefBHsUACGE21gAblmvvPKKHnzwQcXExGjVqlVasmSJ3njjjWCPBSDEcGUHwC3rxz/+sTZt2qRTp07prrvu0sSJEzVu3LhgjwUgxBA7AADAaDygDAAAjEbsAAAAoxE7AADAaMQOAAAwGrEDAACMRuwAAACjETsAAMBoxA4AADDa/wNO7zXh6bBUHAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='diagnosis', data = df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4eab1fd9-7d60-4b01-aaae-a8c7fc819a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "diagnosis\n",
       "B    357\n",
       "M    212\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.diagnosis.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e31a2d10-2bc6-4d00-89af-d9f7940ed41b",
   "metadata": {},
   "source": [
    "# Train Test Split "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "84d09403-dc51-472b-95db-9691d66dbc64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>radius_mean</th>\n",
       "      <th>texture_mean</th>\n",
       "      <th>perimeter_mean</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>smoothness_mean</th>\n",
       "      <th>compactness_mean</th>\n",
       "      <th>concavity_mean</th>\n",
       "      <th>concave points_mean</th>\n",
       "      <th>symmetry_mean</th>\n",
       "      <th>fractal_dimension_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>radius_worst</th>\n",
       "      <th>texture_worst</th>\n",
       "      <th>perimeter_worst</th>\n",
       "      <th>area_worst</th>\n",
       "      <th>smoothness_worst</th>\n",
       "      <th>compactness_worst</th>\n",
       "      <th>concavity_worst</th>\n",
       "      <th>concave points_worst</th>\n",
       "      <th>symmetry_worst</th>\n",
       "      <th>fractal_dimension_worst</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17.99</td>\n",
       "      <td>10.38</td>\n",
       "      <td>122.80</td>\n",
       "      <td>1001.0</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>0.27760</td>\n",
       "      <td>0.3001</td>\n",
       "      <td>0.14710</td>\n",
       "      <td>0.2419</td>\n",
       "      <td>0.07871</td>\n",
       "      <td>...</td>\n",
       "      <td>25.38</td>\n",
       "      <td>17.33</td>\n",
       "      <td>184.60</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>0.1622</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.7119</td>\n",
       "      <td>0.2654</td>\n",
       "      <td>0.4601</td>\n",
       "      <td>0.11890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.57</td>\n",
       "      <td>17.77</td>\n",
       "      <td>132.90</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>0.08474</td>\n",
       "      <td>0.07864</td>\n",
       "      <td>0.0869</td>\n",
       "      <td>0.07017</td>\n",
       "      <td>0.1812</td>\n",
       "      <td>0.05667</td>\n",
       "      <td>...</td>\n",
       "      <td>24.99</td>\n",
       "      <td>23.41</td>\n",
       "      <td>158.80</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>0.1238</td>\n",
       "      <td>0.1866</td>\n",
       "      <td>0.2416</td>\n",
       "      <td>0.1860</td>\n",
       "      <td>0.2750</td>\n",
       "      <td>0.08902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.69</td>\n",
       "      <td>21.25</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1203.0</td>\n",
       "      <td>0.10960</td>\n",
       "      <td>0.15990</td>\n",
       "      <td>0.1974</td>\n",
       "      <td>0.12790</td>\n",
       "      <td>0.2069</td>\n",
       "      <td>0.05999</td>\n",
       "      <td>...</td>\n",
       "      <td>23.57</td>\n",
       "      <td>25.53</td>\n",
       "      <td>152.50</td>\n",
       "      <td>1709.0</td>\n",
       "      <td>0.1444</td>\n",
       "      <td>0.4245</td>\n",
       "      <td>0.4504</td>\n",
       "      <td>0.2430</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.08758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11.42</td>\n",
       "      <td>20.38</td>\n",
       "      <td>77.58</td>\n",
       "      <td>386.1</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.28390</td>\n",
       "      <td>0.2414</td>\n",
       "      <td>0.10520</td>\n",
       "      <td>0.2597</td>\n",
       "      <td>0.09744</td>\n",
       "      <td>...</td>\n",
       "      <td>14.91</td>\n",
       "      <td>26.50</td>\n",
       "      <td>98.87</td>\n",
       "      <td>567.7</td>\n",
       "      <td>0.2098</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.6869</td>\n",
       "      <td>0.2575</td>\n",
       "      <td>0.6638</td>\n",
       "      <td>0.17300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20.29</td>\n",
       "      <td>14.34</td>\n",
       "      <td>135.10</td>\n",
       "      <td>1297.0</td>\n",
       "      <td>0.10030</td>\n",
       "      <td>0.13280</td>\n",
       "      <td>0.1980</td>\n",
       "      <td>0.10430</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.05883</td>\n",
       "      <td>...</td>\n",
       "      <td>22.54</td>\n",
       "      <td>16.67</td>\n",
       "      <td>152.20</td>\n",
       "      <td>1575.0</td>\n",
       "      <td>0.1374</td>\n",
       "      <td>0.2050</td>\n",
       "      <td>0.4000</td>\n",
       "      <td>0.1625</td>\n",
       "      <td>0.2364</td>\n",
       "      <td>0.07678</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   radius_mean  texture_mean  perimeter_mean  area_mean  smoothness_mean  \\\n",
       "0        17.99         10.38          122.80     1001.0          0.11840   \n",
       "1        20.57         17.77          132.90     1326.0          0.08474   \n",
       "2        19.69         21.25          130.00     1203.0          0.10960   \n",
       "3        11.42         20.38           77.58      386.1          0.14250   \n",
       "4        20.29         14.34          135.10     1297.0          0.10030   \n",
       "\n",
       "   compactness_mean  concavity_mean  concave points_mean  symmetry_mean  \\\n",
       "0           0.27760          0.3001              0.14710         0.2419   \n",
       "1           0.07864          0.0869              0.07017         0.1812   \n",
       "2           0.15990          0.1974              0.12790         0.2069   \n",
       "3           0.28390          0.2414              0.10520         0.2597   \n",
       "4           0.13280          0.1980              0.10430         0.1809   \n",
       "\n",
       "   fractal_dimension_mean  ...  radius_worst  texture_worst  perimeter_worst  \\\n",
       "0                 0.07871  ...         25.38          17.33           184.60   \n",
       "1                 0.05667  ...         24.99          23.41           158.80   \n",
       "2                 0.05999  ...         23.57          25.53           152.50   \n",
       "3                 0.09744  ...         14.91          26.50            98.87   \n",
       "4                 0.05883  ...         22.54          16.67           152.20   \n",
       "\n",
       "   area_worst  smoothness_worst  compactness_worst  concavity_worst  \\\n",
       "0      2019.0            0.1622             0.6656           0.7119   \n",
       "1      1956.0            0.1238             0.1866           0.2416   \n",
       "2      1709.0            0.1444             0.4245           0.4504   \n",
       "3       567.7            0.2098             0.8663           0.6869   \n",
       "4      1575.0            0.1374             0.2050           0.4000   \n",
       "\n",
       "   concave points_worst  symmetry_worst  fractal_dimension_worst  \n",
       "0                0.2654          0.4601                  0.11890  \n",
       "1                0.1860          0.2750                  0.08902  \n",
       "2                0.2430          0.3613                  0.08758  \n",
       "3                0.2575          0.6638                  0.17300  \n",
       "4                0.1625          0.2364                  0.07678  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = df.drop('diagnosis', axis = 1)\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83208515-01f3-4f94-a117-60d25a695a44",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>diagnosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  diagnosis\n",
       "0         M\n",
       "1         M\n",
       "2         M\n",
       "3         M\n",
       "4         M"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = df [['diagnosis']]\n",
    "y.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d356bb02-d7a3-4d43-ac5a-00c087db71c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain,xtest,ytrain,ytest = tts(x,y,test_size = .3, random_state = 42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff67951-f45f-48b6-a9e9-ae7144f4bb8c",
   "metadata": {},
   "source": [
    "# Decision Tree Implement & Evaluation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "59bf7de8-36bb-4f6f-95f8-88723106e502",
   "metadata": {},
   "outputs": [],
   "source": [
    "dt_model = DecisionTreeClassifier(random_state = 42)\n",
    "dt_model.fit(xtrain,ytrain)\n",
    "dt_ypred = dt_model.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d2565569-0d13-4891-9ba8-19f0b7fcf518",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['B', 'M', 'M', 'B', 'B', 'M', 'M', 'M', 'B', 'B', 'M', 'M', 'B',\n",
       "       'M', 'B', 'M', 'B', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'B', 'B',\n",
       "       'B', 'B', 'B', 'M', 'B', 'B', 'B', 'B', 'B', 'B', 'M', 'B', 'M',\n",
       "       'B', 'B', 'M', 'B', 'B', 'B', 'B', 'M', 'M', 'B', 'B', 'M', 'M',\n",
       "       'B', 'B', 'B', 'B', 'B', 'M', 'M', 'B', 'B', 'M', 'M', 'B', 'B',\n",
       "       'B', 'M', 'M', 'B', 'B', 'M', 'M', 'B', 'M', 'B', 'B', 'B', 'M',\n",
       "       'B', 'B', 'M', 'B', 'B', 'M', 'M', 'M', 'M', 'M', 'B', 'B', 'B',\n",
       "       'B', 'M', 'B', 'B', 'B', 'M', 'M', 'B', 'M', 'M', 'B', 'M', 'M',\n",
       "       'B', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'M', 'B',\n",
       "       'B', 'B', 'M', 'M', 'B', 'B', 'M', 'B', 'M', 'M', 'B', 'B', 'M',\n",
       "       'M', 'M', 'B', 'B', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'M', 'B',\n",
       "       'B', 'M', 'B', 'M', 'M', 'M', 'B', 'M', 'B', 'B', 'B', 'M', 'M',\n",
       "       'M', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'M', 'B', 'B', 'B', 'B',\n",
       "       'M', 'B'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_ypred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "532d1c6e-4131-4c9c-9c3d-1bd61b0893ef",
   "metadata": {},
   "source": [
    "**Displaying model performance**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "353fc26c-24fa-4a72-a4ec-d5ca6c23457a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree performance:\n",
      "Accuracy: 0.9415204678362573\n"
     ]
    }
   ],
   "source": [
    "print('Decision Tree performance:')\n",
    "print('Accuracy:', accuracy_score(ytest,dt_ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e52c02c2-e51e-422b-96cb-3e01de6d7c07",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Model:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           B       0.97      0.94      0.95       108\n",
      "           M       0.90      0.95      0.92        63\n",
      "\n",
      "    accuracy                           0.94       171\n",
      "   macro avg       0.93      0.94      0.94       171\n",
      "weighted avg       0.94      0.94      0.94       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification Model:')\n",
    "print(classification_report(ytest,dt_ypred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92d3655e-2178-4ced-bad2-5db16bc278cd",
   "metadata": {},
   "source": [
    "# Random forest model Implement & Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4d1a99c5-ebf8-4cf7-ad7f-845eeebcd284",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_model = RandomForestClassifier(random_state = 42)\n",
    "rf_model.fit(xtrain,ytrain)\n",
    "rf_ypred = rf_model.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d419c472-3dba-4733-afc2-364d3d7466aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['B', 'M', 'M', 'B', 'B', 'M', 'M', 'M', 'M', 'B', 'B', 'M', 'B',\n",
       "       'M', 'B', 'M', 'B', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'B', 'B',\n",
       "       'B', 'B', 'B', 'M', 'B', 'B', 'B', 'B', 'B', 'B', 'M', 'B', 'M',\n",
       "       'B', 'B', 'M', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'M', 'M',\n",
       "       'B', 'B', 'B', 'B', 'B', 'M', 'M', 'B', 'B', 'M', 'M', 'B', 'B',\n",
       "       'B', 'M', 'M', 'B', 'B', 'M', 'M', 'B', 'M', 'B', 'B', 'B', 'B',\n",
       "       'B', 'B', 'M', 'B', 'B', 'M', 'M', 'M', 'M', 'M', 'B', 'B', 'B',\n",
       "       'B', 'B', 'B', 'B', 'B', 'M', 'M', 'B', 'M', 'M', 'B', 'M', 'M',\n",
       "       'B', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'B', 'M', 'B', 'M', 'B',\n",
       "       'B', 'B', 'M', 'B', 'B', 'B', 'M', 'B', 'M', 'M', 'B', 'B', 'M',\n",
       "       'M', 'M', 'B', 'B', 'B', 'M', 'B', 'B', 'B', 'M', 'B', 'M', 'B',\n",
       "       'B', 'M', 'B', 'M', 'M', 'M', 'B', 'M', 'B', 'B', 'B', 'B', 'M',\n",
       "       'M', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B', 'B',\n",
       "       'M', 'B'], dtype=object)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_ypred"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae8ef6b6-d063-4c23-8a9e-d1875f98ade1",
   "metadata": {},
   "source": [
    "**Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c6c034dc-cdf0-4863-a1bd-8c1f59a88215",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Model Performance\n",
      "Accuracy: 0.9707602339181286\n"
     ]
    }
   ],
   "source": [
    "print('Random Forest Model Performance')\n",
    "print('Accuracy:', accuracy_score(ytest,rf_ypred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e6862a7f-acee-4a8b-943c-0c898b38584a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.96      0.99      0.98       108\n",
      "           M       0.98      0.94      0.96        63\n",
      "\n",
      "    accuracy                           0.97       171\n",
      "   macro avg       0.97      0.96      0.97       171\n",
      "weighted avg       0.97      0.97      0.97       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification Report\\n',classification_report(ytest,rf_ypred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07464799-5e53-45fb-b40b-85aa7aeb8eb2",
   "metadata": {},
   "source": [
    "# Pre-Pruning for Cost Complexity"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c355ab0b-56e7-4baa-9653-642555948abd",
   "metadata": {},
   "source": [
    "**hyperparameters initialization / optimization**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "901f94a3-69e6-4408-a089-a7dfc25139ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#decision tree parameters optimization\n",
    "\"\"\"\"\n",
    "class sklearn.tree.DecisionTreeClassifier\n",
    "(*, criterion='gini', splitter='best', max_depth=None, \n",
    "min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, \n",
    "max_features=None, random_state=None, max_leaf_nodes=None, \n",
    "min_impurity_decrease=0.0, class_weight=None, ccp_alpha=0.0)\n",
    "\n",
    "\"\"\"\n",
    "dt_params = {\n",
    "    'criterion' : ['gini','entropy','log_loss'] ,\n",
    "    'splitter' : ['best','random'],\n",
    "    'max_depth' : [None,10,20,30],\n",
    "    'min_samples_split' : [2, 5, 10],\n",
    "    'min_samples_leaf' : [1, 2, 4],\n",
    "    \n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "235d75dc-b850-494a-b14d-c7082d8c17e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random forest parameters optimization\n",
    "'''\n",
    "class sklearn.ensemble.RandomForestClassifier\n",
    "(n_estimators=100, *, criterion='gini', max_depth=None,\n",
    "min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, \n",
    "max_features='sqrt', max_leaf_nodes=None, min_impurity_decrease=0.0, \n",
    "bootstrap=True, oob_score=False, n_jobs=None, random_state=None, \n",
    "verbose=0, warm_start=False, class_weight=None, ccp_alpha=0.0, \n",
    "max_samples=None)\n",
    "\n",
    "'''\n",
    "\n",
    "rf_params = {\n",
    "    'n_estimators': [100, 200, 300], #https://numpy.org/doc/stable/reference/random/generated/numpy.random.randint.html\n",
    "    'criterion': ['gini', 'entropy','log_loss'],\n",
    "    'max_depth': [None, 10, 20, 30],\n",
    "    'min_samples_split': [2, 5, 10],\n",
    "    'min_samples_leaf': [1, 2, 4],\n",
    "    'max_features': [None, 'sqrt', 'log2']\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c511e900-50c1-430a-abcd-735c08340f02",
   "metadata": {},
   "source": [
    "# Perform Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b0698b9-9310-4349-aead-61aed9e363bb",
   "metadata": {},
   "source": [
    "**Grid Search for Decision tree**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f0526884-ce01-421c-9b8e-74cefb33fbe5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "             param_grid={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;, &#x27;log_loss&#x27;],\n",
       "                         &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                         &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                         &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                         &#x27;splitter&#x27;: [&#x27;best&#x27;, &#x27;random&#x27;]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "             param_grid={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;, &#x27;log_loss&#x27;],\n",
       "                         &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                         &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                         &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                         &#x27;splitter&#x27;: [&#x27;best&#x27;, &#x27;random&#x27;]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "             param_grid={'criterion': ['gini', 'entropy', 'log_loss'],\n",
       "                         'max_depth': [None, 10, 20, 30],\n",
       "                         'min_samples_leaf': [1, 2, 4],\n",
       "                         'min_samples_split': [2, 5, 10],\n",
       "                         'splitter': ['best', 'random']})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_grid_search = GridSearchCV(estimator = DecisionTreeClassifier(random_state = 42),\n",
    "                             param_grid = dt_params, cv = 5)\n",
    "dt_grid_search.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f6971ccc-280e-4eb0-a6ce-e5a95f99d71b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.01830578, 0.        , 0.01562762, 0.        , 0.01875038,\n",
       "        0.00624943, 0.01562414, 0.00937557, 0.01562476, 0.00625033,\n",
       "        0.01562433, 0.        , 0.01829824, 0.00937715, 0.01562414,\n",
       "        0.00624943, 0.0124999 , 0.00625043, 0.01250005, 0.00312467,\n",
       "        0.01562486, 0.00625005, 0.01562552, 0.0031249 , 0.01562471,\n",
       "        0.0062501 , 0.01672196, 0.00624995, 0.01562462, 0.00560465,\n",
       "        0.0124989 , 0.        , 0.015625  , 0.00937557, 0.00937433,\n",
       "        0.0062499 , 0.01562548, 0.        , 0.0156251 , 0.00312533,\n",
       "        0.01596546, 0.00312624, 0.01562419, 0.00625072, 0.01992788,\n",
       "        0.00312457, 0.01562576, 0.01249928, 0.01249943, 0.0062501 ,\n",
       "        0.00624928, 0.00624952, 0.01250072, 0.0031249 , 0.01527476,\n",
       "        0.        , 0.01802459, 0.00312605, 0.01244335, 0.0047637 ,\n",
       "        0.01277776, 0.        , 0.01722283, 0.00312481, 0.01874957,\n",
       "        0.00937562, 0.01249933, 0.00625005, 0.01532865, 0.00302238,\n",
       "        0.01250124, 0.        , 0.01562557, 0.00312486, 0.01562519,\n",
       "        0.003125  , 0.01249986, 0.00624995, 0.01562543, 0.0062501 ,\n",
       "        0.01250005, 0.00937481, 0.0150898 , 0.00625   , 0.01464658,\n",
       "        0.00675993, 0.01236529, 0.00539961, 0.01719723, 0.00420046,\n",
       "        0.01599612, 0.0024004 , 0.01699705, 0.00420051, 0.01749001,\n",
       "        0.00625057, 0.01562457, 0.        , 0.01505752, 0.00419974,\n",
       "        0.01638093, 0.0045989 , 0.01250124, 0.        , 0.01562524,\n",
       "        0.00937495, 0.01562486, 0.        , 0.017981  , 0.00179982,\n",
       "        0.01249862, 0.00625   , 0.01993146, 0.00937514, 0.01562471,\n",
       "        0.00624943, 0.01250129, 0.00312533, 0.01562481, 0.00624976,\n",
       "        0.01562514, 0.00312495, 0.01250014, 0.00466781, 0.01491308,\n",
       "        0.00937495, 0.01478448, 0.00359998, 0.01519732, 0.00499892,\n",
       "        0.01599894, 0.00460005, 0.01595993, 0.00971174, 0.01669073,\n",
       "        0.00329237, 0.01434598, 0.00706573, 0.0187501 , 0.00413022,\n",
       "        0.01947927, 0.0042593 , 0.01801271, 0.00312552, 0.01562452,\n",
       "        0.00625014, 0.0125    , 0.00742755, 0.01562452, 0.00141463,\n",
       "        0.01463308, 0.00624943, 0.01875548, 0.01665001, 0.01987519,\n",
       "        0.00610733, 0.00802798, 0.00624962, 0.00937581, 0.00850759,\n",
       "        0.01683402, 0.00021439, 0.01457849, 0.00624981, 0.01875024,\n",
       "        0.0015872 , 0.01682377, 0.00674758, 0.00913677, 0.00625029,\n",
       "        0.01562471, 0.00625   , 0.01712985, 0.00662379, 0.01655436,\n",
       "        0.00576801, 0.0125001 , 0.00624952, 0.01249952, 0.        ,\n",
       "        0.01443992, 0.00312529, 0.01562481, 0.        , 0.00957637,\n",
       "        0.        , 0.01368389, 0.00577488, 0.01126633, 0.00624971,\n",
       "        0.01876245, 0.00344801, 0.01658263, 0.00826888, 0.01065283,\n",
       "        0.00624962, 0.01562452, 0.00937529, 0.01611414, 0.        ,\n",
       "        0.0156249 , 0.00723548, 0.0199882 , 0.00347528, 0.01852965,\n",
       "        0.00624962, 0.01562753, 0.00625   , 0.01301064, 0.0057951 ,\n",
       "        0.01562586, 0.00624981, 0.01250019, 0.        , 0.01441455,\n",
       "        0.00079999]),\n",
       " 'std_fit_time': array([2.13386016e-03, 0.00000000e+00, 4.54373522e-06, 0.00000000e+00,\n",
       "        6.25112078e-03, 7.65395467e-03, 1.14068777e-05, 7.65512272e-03,\n",
       "        1.87730977e-06, 7.65506431e-03, 1.11216581e-06, 0.00000000e+00,\n",
       "        7.86338625e-03, 7.65640777e-03, 3.24669734e-06, 7.65395467e-03,\n",
       "        6.24995239e-03, 7.65518108e-03, 6.25002389e-03, 6.24933243e-03,\n",
       "        1.74290341e-06, 7.65471386e-03, 1.40969710e-06, 6.24980927e-03,\n",
       "        9.81867804e-07, 7.65477225e-03, 1.20469371e-02, 7.65459705e-03,\n",
       "        8.06404806e-07, 8.53192940e-03, 6.24945174e-03, 0.00000000e+00,\n",
       "        1.65868064e-06, 7.65512317e-03, 7.65411039e-03, 7.65453865e-03,\n",
       "        2.76401076e-06, 0.00000000e+00, 4.90933902e-07, 6.25066757e-03,\n",
       "        1.80658618e-03, 6.25247955e-03, 2.68726402e-06, 7.65553147e-03,\n",
       "        6.09907783e-03, 6.24914169e-03, 2.48413396e-06, 6.24964245e-03,\n",
       "        6.24971394e-03, 7.65477225e-03, 7.65377946e-03, 7.65407144e-03,\n",
       "        6.25035770e-03, 6.24980927e-03, 2.29415532e-03, 0.00000000e+00,\n",
       "        2.31408922e-03, 6.25209808e-03, 6.22264728e-03, 1.93313582e-03,\n",
       "        5.69412724e-03, 0.00000000e+00, 7.37062731e-03, 6.24961853e-03,\n",
       "        6.25033408e-03, 7.65516166e-03, 6.24966662e-03, 7.65471391e-03,\n",
       "        2.36157396e-03, 1.54933634e-03, 6.25062012e-03, 0.00000000e+00,\n",
       "        9.88287167e-03, 6.24971390e-03, 8.03580262e-07, 6.25000000e-03,\n",
       "        6.24992857e-03, 7.65459706e-03, 5.51978917e-07, 7.65477225e-03,\n",
       "        6.25002389e-03, 7.65449977e-03, 2.68556781e-03, 7.65465557e-03,\n",
       "        5.92139511e-03, 3.68831277e-03, 5.46403125e-03, 7.99465706e-04,\n",
       "        2.56023377e-03, 3.99352102e-04, 2.28033231e-03, 1.95991727e-03,\n",
       "        2.60559075e-03, 4.00521369e-04, 6.79590367e-03, 7.65535640e-03,\n",
       "        5.30983387e-07, 0.00000000e+00, 6.95204403e-03, 3.99828028e-04,\n",
       "        3.48838598e-03, 2.49740974e-03, 6.25062215e-03, 0.00000000e+00,\n",
       "        1.66552059e-06, 7.65461653e-03, 9.28008908e-06, 0.00000000e+00,\n",
       "        3.55592869e-03, 2.22683302e-03, 6.24930872e-03, 7.65465566e-03,\n",
       "        7.40795895e-03, 7.65477228e-03, 9.81867804e-07, 7.65395468e-03,\n",
       "        6.25064508e-03, 6.25066757e-03, 6.64157308e-07, 7.65436345e-03,\n",
       "        1.96373561e-06, 6.24990463e-03, 6.25007159e-03, 3.87998991e-03,\n",
       "        1.42681803e-03, 7.65461656e-03, 3.13857244e-03, 1.85461518e-03,\n",
       "        2.03900692e-03, 8.93030300e-04, 2.68340800e-03, 4.89497649e-04,\n",
       "        1.71475056e-03, 7.96224599e-03, 3.59738290e-04, 6.58473969e-03,\n",
       "        8.88765590e-03, 7.14824285e-03, 6.25078799e-03, 7.34532536e-03,\n",
       "        6.69460974e-03, 6.30086381e-03, 5.68085966e-03, 6.25104904e-03,\n",
       "        1.02270177e-06, 7.65483069e-03, 6.25000013e-03, 7.19581866e-03,\n",
       "        1.21570099e-06, 2.82926559e-03, 1.98538347e-03, 7.65395471e-03,\n",
       "        5.26504342e-03, 1.51965936e-02, 5.34223003e-03, 7.48688526e-03,\n",
       "        6.89504624e-03, 7.65418824e-03, 7.65531763e-03, 7.13607983e-03,\n",
       "        1.50909074e-03, 4.28771973e-04, 5.86740447e-03, 7.65442186e-03,\n",
       "        6.25250374e-03, 3.17440033e-03, 3.81196729e-04, 8.32734832e-03,\n",
       "        7.10825391e-03, 7.65500585e-03, 1.56268784e-06, 7.65465547e-03,\n",
       "        3.73958676e-03, 8.11364858e-03, 7.24912935e-04, 6.15486725e-03,\n",
       "        6.25004807e-03, 7.65407152e-03, 6.24976159e-03, 0.00000000e+00,\n",
       "        2.82342390e-03, 6.25057220e-03, 1.84308511e-06, 0.00000000e+00,\n",
       "        7.82771070e-03, 0.00000000e+00, 5.75062512e-03, 6.91959355e-03,\n",
       "        6.19782249e-03, 7.65430507e-03, 6.31258954e-03, 6.89601898e-03,\n",
       "        6.06558919e-04, 6.67916888e-03, 6.41674895e-03, 7.65418827e-03,\n",
       "        6.03156597e-07, 7.65488907e-03, 3.75504088e-03, 0.00000000e+00,\n",
       "        1.28480850e-06, 6.99884048e-03, 6.67331949e-03, 6.95056915e-03,\n",
       "        6.41256258e-03, 7.65418846e-03, 7.14715118e-06, 7.65465545e-03,\n",
       "        6.58023566e-03, 6.23774671e-03, 9.88483232e-03, 7.65442188e-03,\n",
       "        6.25009551e-03, 0.00000000e+00, 8.10541335e-03, 1.59997940e-03]),\n",
       " 'mean_score_time': array([0.00443969, 0.00625086, 0.00625052, 0.00624886, 0.        ,\n",
       "        0.00312548, 0.00312533, 0.        , 0.        , 0.00312467,\n",
       "        0.00312533, 0.009375  , 0.00160074, 0.        , 0.00312524,\n",
       "        0.        , 0.00625019, 0.        , 0.00625033, 0.00624981,\n",
       "        0.0031251 , 0.        , 0.00625052, 0.00624928, 0.00312514,\n",
       "        0.        , 0.00492725, 0.        , 0.0031251 , 0.00883765,\n",
       "        0.006251  , 0.00937529, 0.        , 0.        , 0.00625005,\n",
       "        0.        , 0.00624971, 0.00624971, 0.        , 0.00937538,\n",
       "        0.00080018, 0.00625029, 0.003125  , 0.003125  , 0.00120039,\n",
       "        0.00625048, 0.0062501 , 0.        , 0.00625019, 0.00312567,\n",
       "        0.00937452, 0.        , 0.00312424, 0.0062501 , 0.00336967,\n",
       "        0.00625   , 0.00601168, 0.00312471, 0.00725012, 0.00320024,\n",
       "        0.00625105, 0.00624952, 0.00392642, 0.00312514, 0.        ,\n",
       "        0.        , 0.00625005, 0.        , 0.00426779, 0.00634837,\n",
       "        0.0093749 , 0.00937562, 0.00624919, 0.00624924, 0.00312529,\n",
       "        0.00312452, 0.00625   , 0.00312524, 0.00312457, 0.        ,\n",
       "        0.00625   , 0.        , 0.00302711, 0.0031251 , 0.00621943,\n",
       "        0.00520039, 0.00821624, 0.00479994, 0.00540562, 0.00459957,\n",
       "        0.00480261, 0.00633206, 0.00580306, 0.00499992, 0.00498428,\n",
       "        0.00312519, 0.00312505, 0.00937581, 0.00786791, 0.0048008 ,\n",
       "        0.00320234, 0.00416365, 0.00624962, 0.00937462, 0.        ,\n",
       "        0.        , 0.00312486, 0.00625052, 0.00412526, 0.00454116,\n",
       "        0.00312529, 0.00312524, 0.00160031, 0.        , 0.00312567,\n",
       "        0.        , 0.00624862, 0.00625005, 0.        , 0.00312562,\n",
       "        0.00312471, 0.00312467, 0.00625014, 0.00259948, 0.        ,\n",
       "        0.        , 0.00812001, 0.00660577, 0.00540261, 0.00479999,\n",
       "        0.0050014 , 0.0042007 , 0.00240102, 0.00364785, 0.00336905,\n",
       "        0.00639987, 0.00788298, 0.        , 0.        , 0.00760527,\n",
       "        0.00353079, 0.00347395, 0.0010004 , 0.00624986, 0.00312486,\n",
       "        0.        , 0.00312481, 0.00625815, 0.00625162, 0.00757647,\n",
       "        0.00312514, 0.        , 0.00621543, 0.00333085, 0.00341053,\n",
       "        0.00388026, 0.01154861, 0.        , 0.00937533, 0.0038527 ,\n",
       "        0.00311389, 0.00963387, 0.00772033, 0.        , 0.        ,\n",
       "        0.01402478, 0.00673313, 0.0050128 , 0.00785146, 0.00312476,\n",
       "        0.00312495, 0.00312581, 0.00601888, 0.00334797, 0.0067142 ,\n",
       "        0.00348935, 0.00625043, 0.        , 0.00625029, 0.00937567,\n",
       "        0.00492611, 0.00937448, 0.        , 0.00624995, 0.01417131,\n",
       "        0.00995183, 0.00647998, 0.00444312, 0.00745168, 0.00312533,\n",
       "        0.00246534, 0.00989304, 0.00345364, 0.00300012, 0.00625024,\n",
       "        0.00312552, 0.        , 0.        , 0.00401402, 0.00624962,\n",
       "        0.00312533, 0.00139985, 0.00331025, 0.00980225, 0.00392528,\n",
       "        0.        , 0.00312185, 0.00312543, 0.00686932, 0.00402102,\n",
       "        0.00624995, 0.        , 0.0062499 , 0.00624943, 0.00392585,\n",
       "        0.00719471]),\n",
       " 'std_score_time': array([0.00286182, 0.00765571, 0.0076553 , 0.00765325, 0.        ,\n",
       "        0.00625095, 0.00625067, 0.        , 0.        , 0.00624933,\n",
       "        0.00625067, 0.00765466, 0.0019605 , 0.        , 0.00625048,\n",
       "        0.        , 0.00765489, 0.        , 0.00765506, 0.00765442,\n",
       "        0.00625019, 0.        , 0.0076553 , 0.00765378, 0.00625029,\n",
       "        0.        , 0.00572744, 0.        , 0.00625019, 0.00603447,\n",
       "        0.00765588, 0.00765489, 0.        , 0.        , 0.00765471,\n",
       "        0.        , 0.00765431, 0.00765431, 0.        , 0.00765497,\n",
       "        0.00160036, 0.00765501, 0.00625   , 0.00625   , 0.00240078,\n",
       "        0.00765524, 0.00765477, 0.        , 0.00765489, 0.00625134,\n",
       "        0.00765427, 0.        , 0.00624847, 0.00765477, 0.00498511,\n",
       "        0.00765466, 0.00555988, 0.00624943, 0.0070773 , 0.00160012,\n",
       "        0.00765594, 0.00765407, 0.0060514 , 0.00625029, 0.        ,\n",
       "        0.        , 0.00765471, 0.        , 0.00671594, 0.0068752 ,\n",
       "        0.00765458, 0.00765516, 0.00765366, 0.00765372, 0.00625057,\n",
       "        0.00624905, 0.00765466, 0.00625048, 0.00624914, 0.        ,\n",
       "        0.00765466, 0.        , 0.0043388 , 0.00625019, 0.00577484,\n",
       "        0.00116606, 0.00440183, 0.00074867, 0.00102463, 0.00048969,\n",
       "        0.00039978, 0.00639563, 0.00116552, 0.00063347, 0.00614435,\n",
       "        0.00625038, 0.0062501 , 0.00765532, 0.00734488, 0.0007496 ,\n",
       "        0.00271525, 0.00309839, 0.00765419, 0.00765434, 0.        ,\n",
       "        0.        , 0.00624971, 0.0076553 , 0.00606695, 0.00577452,\n",
       "        0.00625057, 0.00625048, 0.00320063, 0.        , 0.00625134,\n",
       "        0.        , 0.00765296, 0.00765471, 0.        , 0.00625124,\n",
       "        0.00624943, 0.00624933, 0.00765483, 0.00215356, 0.        ,\n",
       "        0.        , 0.00844654, 0.00537408, 0.0010201 , 0.0003999 ,\n",
       "        0.0010962 , 0.00039935, 0.00300792, 0.0064778 , 0.00673809,\n",
       "        0.00787487, 0.00626669, 0.        , 0.        , 0.00688708,\n",
       "        0.00706158, 0.0069479 , 0.00200081, 0.00765448, 0.00624971,\n",
       "        0.        , 0.00624962, 0.00417414, 0.00765664, 0.00782871,\n",
       "        0.00625029, 0.        , 0.00761251, 0.00596493, 0.00682106,\n",
       "        0.00625839, 0.00707433, 0.        , 0.00765493, 0.00671071,\n",
       "        0.00526544, 0.00789593, 0.00576954, 0.        , 0.        ,\n",
       "        0.00339741, 0.00824639, 0.00648066, 0.00651445, 0.00624952,\n",
       "        0.0062499 , 0.00625162, 0.00545305, 0.00669594, 0.00818762,\n",
       "        0.00296312, 0.00765518, 0.        , 0.00765501, 0.0076552 ,\n",
       "        0.00572628, 0.00765423, 0.        , 0.0076546 , 0.00730516,\n",
       "        0.00812991, 0.00793638, 0.00483647, 0.00702612, 0.00625067,\n",
       "        0.00493069, 0.00812747, 0.00641843, 0.00268305, 0.00765495,\n",
       "        0.00625105, 0.        , 0.        , 0.00553814, 0.00765419,\n",
       "        0.00625067, 0.0027997 , 0.0066205 , 0.00801092, 0.00605193,\n",
       "        0.        , 0.00624371, 0.00625086, 0.00723851, 0.00329991,\n",
       "        0.0076546 , 0.        , 0.00765454, 0.00765395, 0.0060524 ,\n",
       "        0.00710286]),\n",
       " 'param_criterion': masked_array(data=['gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'gini', 'gini', 'gini', 'gini', 'gini',\n",
       "                    'gini', 'gini', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'entropy',\n",
       "                    'entropy', 'entropy', 'entropy', 'entropy', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss', 'log_loss',\n",
       "                    'log_loss', 'log_loss', 'log_loss'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_max_depth': masked_array(data=[None, None, None, None, None, None, None, None, None,\n",
       "                    None, None, None, None, None, None, None, None, None,\n",
       "                    10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "                    10, 10, 10, 10, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,\n",
       "                    20, 20, 20, 20, 20, 20, 20, 20, 30, 30, 30, 30, 30, 30,\n",
       "                    30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, None,\n",
       "                    None, None, None, None, None, None, None, None, None,\n",
       "                    None, None, None, None, None, None, None, None, 10, 10,\n",
       "                    10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "                    10, 10, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,\n",
       "                    20, 20, 20, 20, 20, 20, 30, 30, 30, 30, 30, 30, 30, 30,\n",
       "                    30, 30, 30, 30, 30, 30, 30, 30, 30, 30, None, None,\n",
       "                    None, None, None, None, None, None, None, None, None,\n",
       "                    None, None, None, None, None, None, None, 10, 10, 10,\n",
       "                    10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,\n",
       "                    10, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,\n",
       "                    20, 20, 20, 20, 20, 30, 30, 30, 30, 30, 30, 30, 30, 30,\n",
       "                    30, 30, 30, 30, 30, 30, 30, 30, 30],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_samples_leaf': masked_array(data=[1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4,\n",
       "                    1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 4, 4],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_samples_split': masked_array(data=[2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10,\n",
       "                    10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5,\n",
       "                    10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2,\n",
       "                    5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10,\n",
       "                    2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10,\n",
       "                    10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5,\n",
       "                    10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2,\n",
       "                    5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10,\n",
       "                    2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10,\n",
       "                    10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5,\n",
       "                    10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2,\n",
       "                    5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10,\n",
       "                    2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10, 10, 2, 2, 5, 5, 10,\n",
       "                    10],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_splitter': masked_array(data=['best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random',\n",
       "                    'best', 'random', 'best', 'random', 'best', 'random'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'gini',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'entropy',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': None,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 10,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 20,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 1,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 2,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 2,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 5,\n",
       "   'splitter': 'random'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'best'},\n",
       "  {'criterion': 'log_loss',\n",
       "   'max_depth': 30,\n",
       "   'min_samples_leaf': 4,\n",
       "   'min_samples_split': 10,\n",
       "   'splitter': 'random'}],\n",
       " 'split0_test_score': array([0.9125, 0.9375, 0.9125, 0.925 , 0.9375, 0.9375, 0.9125, 0.875 ,\n",
       "        0.9125, 0.9375, 0.9375, 0.925 , 0.95  , 0.925 , 0.95  , 0.925 ,\n",
       "        0.9375, 0.925 , 0.9125, 0.9375, 0.9125, 0.925 , 0.9375, 0.9375,\n",
       "        0.9125, 0.875 , 0.9125, 0.9375, 0.9375, 0.925 , 0.95  , 0.925 ,\n",
       "        0.95  , 0.925 , 0.9375, 0.925 , 0.9125, 0.9375, 0.9125, 0.925 ,\n",
       "        0.9375, 0.9375, 0.9125, 0.875 , 0.9125, 0.9375, 0.9375, 0.925 ,\n",
       "        0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 , 0.9125, 0.9375,\n",
       "        0.9125, 0.925 , 0.9375, 0.9375, 0.9125, 0.875 , 0.9125, 0.9375,\n",
       "        0.9375, 0.925 , 0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 ,\n",
       "        0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9375,\n",
       "        0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9125, 0.95  , 0.9125,\n",
       "        0.925 , 0.9125, 0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 ,\n",
       "        0.95  , 0.9375, 0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9125,\n",
       "        0.95  , 0.9125, 0.925 , 0.9125, 0.95  , 0.925 , 0.95  , 0.925 ,\n",
       "        0.9375, 0.925 , 0.95  , 0.9375, 0.95  , 0.925 , 0.9375, 0.925 ,\n",
       "        0.95  , 0.9125, 0.95  , 0.9125, 0.925 , 0.9125, 0.95  , 0.925 ,\n",
       "        0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9375, 0.95  , 0.925 ,\n",
       "        0.9375, 0.925 , 0.95  , 0.9125, 0.95  , 0.9125, 0.925 , 0.9125,\n",
       "        0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9375,\n",
       "        0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9125, 0.95  , 0.9125,\n",
       "        0.925 , 0.9125, 0.95  , 0.925 , 0.95  , 0.925 , 0.9375, 0.925 ,\n",
       "        0.95  , 0.9375, 0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9125,\n",
       "        0.95  , 0.9125, 0.925 , 0.9125, 0.95  , 0.925 , 0.95  , 0.925 ,\n",
       "        0.9375, 0.925 , 0.95  , 0.9375, 0.95  , 0.925 , 0.9375, 0.925 ,\n",
       "        0.95  , 0.9125, 0.95  , 0.9125, 0.925 , 0.9125, 0.95  , 0.925 ,\n",
       "        0.95  , 0.925 , 0.9375, 0.925 , 0.95  , 0.9375, 0.95  , 0.925 ,\n",
       "        0.9375, 0.925 , 0.95  , 0.9125, 0.95  , 0.9125, 0.925 , 0.9125]),\n",
       " 'split1_test_score': array([0.8875, 0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.9125, 0.875 ,\n",
       "        0.9125, 0.9   , 0.9   , 0.8875, 0.9125, 0.875 , 0.9125, 0.875 ,\n",
       "        0.8875, 0.875 , 0.8875, 0.9   , 0.8875, 0.8875, 0.8875, 0.8875,\n",
       "        0.9125, 0.875 , 0.9125, 0.9   , 0.9   , 0.8875, 0.9125, 0.875 ,\n",
       "        0.9125, 0.875 , 0.8875, 0.875 , 0.8875, 0.9   , 0.8875, 0.8875,\n",
       "        0.8875, 0.8875, 0.9125, 0.875 , 0.9125, 0.9   , 0.9   , 0.8875,\n",
       "        0.9125, 0.875 , 0.9125, 0.875 , 0.8875, 0.875 , 0.8875, 0.9   ,\n",
       "        0.8875, 0.8875, 0.8875, 0.8875, 0.9125, 0.875 , 0.9125, 0.9   ,\n",
       "        0.9   , 0.8875, 0.9125, 0.875 , 0.9125, 0.875 , 0.8875, 0.875 ,\n",
       "        0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.9   , 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.875 , 0.8875, 0.875 ,\n",
       "        0.8875, 0.875 , 0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875,\n",
       "        0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.875 ,\n",
       "        0.8875, 0.875 , 0.8875, 0.875 , 0.9   , 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.8875, 0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.875 , 0.8875, 0.875 , 0.8875, 0.875 , 0.9   , 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.8875, 0.9   , 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.875 , 0.8875, 0.875 , 0.8875, 0.875 ,\n",
       "        0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.9   , 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.875 , 0.8875, 0.875 ,\n",
       "        0.8875, 0.875 , 0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875,\n",
       "        0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.8875, 0.875 ,\n",
       "        0.8875, 0.875 , 0.8875, 0.875 , 0.9   , 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.8875, 0.9   , 0.8875, 0.8875, 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.875 , 0.8875, 0.875 , 0.8875, 0.875 , 0.9   , 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.8875, 0.9   , 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.8875, 0.8875, 0.875 , 0.8875, 0.875 , 0.8875, 0.875 ]),\n",
       " 'split2_test_score': array([0.9   , 0.9625, 0.925 , 0.9625, 0.925 , 0.95  , 0.925 , 0.9625,\n",
       "        0.9375, 0.9625, 0.9125, 0.9625, 0.925 , 0.9375, 0.925 , 0.9375,\n",
       "        0.925 , 0.95  , 0.9   , 0.9625, 0.925 , 0.9625, 0.925 , 0.95  ,\n",
       "        0.925 , 0.9625, 0.9375, 0.9625, 0.9125, 0.9625, 0.925 , 0.9375,\n",
       "        0.925 , 0.9375, 0.925 , 0.95  , 0.9   , 0.9625, 0.925 , 0.9625,\n",
       "        0.925 , 0.95  , 0.925 , 0.9625, 0.9375, 0.9625, 0.9125, 0.9625,\n",
       "        0.925 , 0.9375, 0.925 , 0.9375, 0.925 , 0.95  , 0.9   , 0.9625,\n",
       "        0.925 , 0.9625, 0.925 , 0.95  , 0.925 , 0.9625, 0.9375, 0.9625,\n",
       "        0.9125, 0.9625, 0.925 , 0.9375, 0.925 , 0.9375, 0.925 , 0.95  ,\n",
       "        0.95  , 0.925 , 0.95  , 0.9375, 0.975 , 0.9375, 0.95  , 0.8875,\n",
       "        0.95  , 0.9125, 0.975 , 0.95  , 0.9625, 0.9625, 0.9625, 0.9625,\n",
       "        0.9625, 0.925 , 0.95  , 0.925 , 0.95  , 0.9375, 0.975 , 0.9375,\n",
       "        0.95  , 0.8875, 0.95  , 0.9125, 0.975 , 0.95  , 0.9625, 0.9625,\n",
       "        0.9625, 0.9625, 0.9625, 0.925 , 0.95  , 0.925 , 0.95  , 0.9375,\n",
       "        0.975 , 0.9375, 0.95  , 0.8875, 0.95  , 0.9125, 0.975 , 0.95  ,\n",
       "        0.9625, 0.9625, 0.9625, 0.9625, 0.9625, 0.925 , 0.95  , 0.925 ,\n",
       "        0.95  , 0.9375, 0.975 , 0.9375, 0.95  , 0.8875, 0.95  , 0.9125,\n",
       "        0.975 , 0.95  , 0.9625, 0.9625, 0.9625, 0.9625, 0.9625, 0.925 ,\n",
       "        0.95  , 0.925 , 0.95  , 0.9375, 0.975 , 0.9375, 0.95  , 0.8875,\n",
       "        0.95  , 0.9125, 0.975 , 0.95  , 0.9625, 0.9625, 0.9625, 0.9625,\n",
       "        0.9625, 0.925 , 0.95  , 0.925 , 0.95  , 0.9375, 0.975 , 0.9375,\n",
       "        0.95  , 0.8875, 0.95  , 0.9125, 0.975 , 0.95  , 0.9625, 0.9625,\n",
       "        0.9625, 0.9625, 0.9625, 0.925 , 0.95  , 0.925 , 0.95  , 0.9375,\n",
       "        0.975 , 0.9375, 0.95  , 0.8875, 0.95  , 0.9125, 0.975 , 0.95  ,\n",
       "        0.9625, 0.9625, 0.9625, 0.9625, 0.9625, 0.925 , 0.95  , 0.925 ,\n",
       "        0.95  , 0.9375, 0.975 , 0.9375, 0.95  , 0.8875, 0.95  , 0.9125,\n",
       "        0.975 , 0.95  , 0.9625, 0.9625, 0.9625, 0.9625, 0.9625, 0.925 ]),\n",
       " 'split3_test_score': array([0.93670886, 0.91139241, 0.93670886, 0.92405063, 0.91139241,\n",
       "        0.93670886, 0.94936709, 0.93670886, 0.94936709, 0.91139241,\n",
       "        0.92405063, 0.93670886, 0.94936709, 0.93670886, 0.94936709,\n",
       "        0.93670886, 0.92405063, 0.94936709, 0.93670886, 0.91139241,\n",
       "        0.93670886, 0.92405063, 0.91139241, 0.93670886, 0.94936709,\n",
       "        0.93670886, 0.94936709, 0.91139241, 0.92405063, 0.93670886,\n",
       "        0.94936709, 0.93670886, 0.94936709, 0.93670886, 0.92405063,\n",
       "        0.94936709, 0.93670886, 0.91139241, 0.93670886, 0.92405063,\n",
       "        0.91139241, 0.93670886, 0.94936709, 0.93670886, 0.94936709,\n",
       "        0.91139241, 0.92405063, 0.93670886, 0.94936709, 0.93670886,\n",
       "        0.94936709, 0.93670886, 0.92405063, 0.94936709, 0.93670886,\n",
       "        0.91139241, 0.93670886, 0.92405063, 0.91139241, 0.93670886,\n",
       "        0.94936709, 0.93670886, 0.94936709, 0.91139241, 0.92405063,\n",
       "        0.93670886, 0.94936709, 0.93670886, 0.94936709, 0.93670886,\n",
       "        0.92405063, 0.94936709, 0.93670886, 0.96202532, 0.93670886,\n",
       "        0.89873418, 0.94936709, 0.93670886, 0.93670886, 0.93670886,\n",
       "        0.93670886, 0.89873418, 0.94936709, 0.93670886, 0.94936709,\n",
       "        0.94936709, 0.94936709, 0.94936709, 0.94936709, 0.93670886,\n",
       "        0.93670886, 0.96202532, 0.93670886, 0.89873418, 0.94936709,\n",
       "        0.93670886, 0.93670886, 0.93670886, 0.93670886, 0.89873418,\n",
       "        0.94936709, 0.93670886, 0.94936709, 0.94936709, 0.94936709,\n",
       "        0.94936709, 0.94936709, 0.93670886, 0.93670886, 0.96202532,\n",
       "        0.93670886, 0.89873418, 0.94936709, 0.93670886, 0.93670886,\n",
       "        0.93670886, 0.93670886, 0.89873418, 0.94936709, 0.93670886,\n",
       "        0.94936709, 0.94936709, 0.94936709, 0.94936709, 0.94936709,\n",
       "        0.93670886, 0.93670886, 0.96202532, 0.93670886, 0.89873418,\n",
       "        0.94936709, 0.93670886, 0.93670886, 0.93670886, 0.93670886,\n",
       "        0.89873418, 0.94936709, 0.93670886, 0.94936709, 0.94936709,\n",
       "        0.94936709, 0.94936709, 0.94936709, 0.93670886, 0.93670886,\n",
       "        0.96202532, 0.93670886, 0.89873418, 0.94936709, 0.93670886,\n",
       "        0.93670886, 0.93670886, 0.93670886, 0.89873418, 0.94936709,\n",
       "        0.93670886, 0.94936709, 0.94936709, 0.94936709, 0.94936709,\n",
       "        0.94936709, 0.93670886, 0.93670886, 0.96202532, 0.93670886,\n",
       "        0.89873418, 0.94936709, 0.93670886, 0.93670886, 0.93670886,\n",
       "        0.93670886, 0.89873418, 0.94936709, 0.93670886, 0.94936709,\n",
       "        0.94936709, 0.94936709, 0.94936709, 0.94936709, 0.93670886,\n",
       "        0.93670886, 0.96202532, 0.93670886, 0.89873418, 0.94936709,\n",
       "        0.93670886, 0.93670886, 0.93670886, 0.93670886, 0.89873418,\n",
       "        0.94936709, 0.93670886, 0.94936709, 0.94936709, 0.94936709,\n",
       "        0.94936709, 0.94936709, 0.93670886, 0.93670886, 0.96202532,\n",
       "        0.93670886, 0.89873418, 0.94936709, 0.93670886, 0.93670886,\n",
       "        0.93670886, 0.93670886, 0.89873418, 0.94936709, 0.93670886,\n",
       "        0.94936709, 0.94936709, 0.94936709, 0.94936709, 0.94936709,\n",
       "        0.93670886]),\n",
       " 'split4_test_score': array([0.91139241, 0.94936709, 0.93670886, 0.96202532, 0.92405063,\n",
       "        0.93670886, 0.88607595, 0.91139241, 0.92405063, 0.91139241,\n",
       "        0.92405063, 0.93670886, 0.92405063, 0.86075949, 0.92405063,\n",
       "        0.86075949, 0.92405063, 0.86075949, 0.91139241, 0.94936709,\n",
       "        0.93670886, 0.96202532, 0.92405063, 0.93670886, 0.88607595,\n",
       "        0.91139241, 0.92405063, 0.91139241, 0.92405063, 0.93670886,\n",
       "        0.92405063, 0.86075949, 0.92405063, 0.86075949, 0.92405063,\n",
       "        0.86075949, 0.91139241, 0.94936709, 0.93670886, 0.96202532,\n",
       "        0.92405063, 0.93670886, 0.88607595, 0.91139241, 0.92405063,\n",
       "        0.91139241, 0.92405063, 0.93670886, 0.92405063, 0.86075949,\n",
       "        0.92405063, 0.86075949, 0.92405063, 0.86075949, 0.91139241,\n",
       "        0.94936709, 0.93670886, 0.96202532, 0.92405063, 0.93670886,\n",
       "        0.88607595, 0.91139241, 0.92405063, 0.91139241, 0.92405063,\n",
       "        0.93670886, 0.92405063, 0.86075949, 0.92405063, 0.86075949,\n",
       "        0.92405063, 0.86075949, 0.91139241, 0.91139241, 0.92405063,\n",
       "        0.89873418, 0.92405063, 0.93670886, 0.93670886, 0.87341772,\n",
       "        0.93670886, 0.87341772, 0.93670886, 0.93670886, 0.92405063,\n",
       "        0.93670886, 0.92405063, 0.93670886, 0.92405063, 0.86075949,\n",
       "        0.91139241, 0.91139241, 0.92405063, 0.89873418, 0.92405063,\n",
       "        0.93670886, 0.93670886, 0.87341772, 0.93670886, 0.87341772,\n",
       "        0.93670886, 0.93670886, 0.92405063, 0.93670886, 0.92405063,\n",
       "        0.93670886, 0.92405063, 0.86075949, 0.91139241, 0.91139241,\n",
       "        0.92405063, 0.89873418, 0.92405063, 0.93670886, 0.93670886,\n",
       "        0.87341772, 0.93670886, 0.87341772, 0.93670886, 0.93670886,\n",
       "        0.92405063, 0.93670886, 0.92405063, 0.93670886, 0.92405063,\n",
       "        0.86075949, 0.91139241, 0.91139241, 0.92405063, 0.89873418,\n",
       "        0.92405063, 0.93670886, 0.93670886, 0.87341772, 0.93670886,\n",
       "        0.87341772, 0.93670886, 0.93670886, 0.92405063, 0.93670886,\n",
       "        0.92405063, 0.93670886, 0.92405063, 0.86075949, 0.91139241,\n",
       "        0.91139241, 0.92405063, 0.89873418, 0.92405063, 0.93670886,\n",
       "        0.93670886, 0.87341772, 0.93670886, 0.87341772, 0.93670886,\n",
       "        0.93670886, 0.92405063, 0.93670886, 0.92405063, 0.93670886,\n",
       "        0.92405063, 0.86075949, 0.91139241, 0.91139241, 0.92405063,\n",
       "        0.89873418, 0.92405063, 0.93670886, 0.93670886, 0.87341772,\n",
       "        0.93670886, 0.87341772, 0.93670886, 0.93670886, 0.92405063,\n",
       "        0.93670886, 0.92405063, 0.93670886, 0.92405063, 0.86075949,\n",
       "        0.91139241, 0.91139241, 0.92405063, 0.89873418, 0.92405063,\n",
       "        0.93670886, 0.93670886, 0.87341772, 0.93670886, 0.87341772,\n",
       "        0.93670886, 0.93670886, 0.92405063, 0.93670886, 0.92405063,\n",
       "        0.93670886, 0.92405063, 0.86075949, 0.91139241, 0.91139241,\n",
       "        0.92405063, 0.89873418, 0.92405063, 0.93670886, 0.93670886,\n",
       "        0.87341772, 0.93670886, 0.87341772, 0.93670886, 0.93670886,\n",
       "        0.92405063, 0.93670886, 0.92405063, 0.93670886, 0.92405063,\n",
       "        0.86075949]),\n",
       " 'mean_test_score': array([0.90962025, 0.9321519 , 0.91968354, 0.93221519, 0.91708861,\n",
       "        0.92968354, 0.91708861, 0.91212025, 0.92718354, 0.92455696,\n",
       "        0.91962025, 0.92968354, 0.93218354, 0.90699367, 0.93218354,\n",
       "        0.90699367, 0.91962025, 0.91202532, 0.90962025, 0.9321519 ,\n",
       "        0.91968354, 0.93221519, 0.91708861, 0.92968354, 0.91708861,\n",
       "        0.91212025, 0.92718354, 0.92455696, 0.91962025, 0.92968354,\n",
       "        0.93218354, 0.90699367, 0.93218354, 0.90699367, 0.91962025,\n",
       "        0.91202532, 0.90962025, 0.9321519 , 0.91968354, 0.93221519,\n",
       "        0.91708861, 0.92968354, 0.91708861, 0.91212025, 0.92718354,\n",
       "        0.92455696, 0.91962025, 0.92968354, 0.93218354, 0.90699367,\n",
       "        0.93218354, 0.90699367, 0.91962025, 0.91202532, 0.90962025,\n",
       "        0.9321519 , 0.91968354, 0.93221519, 0.91708861, 0.92968354,\n",
       "        0.91708861, 0.91212025, 0.92718354, 0.92455696, 0.91962025,\n",
       "        0.92968354, 0.93218354, 0.90699367, 0.93218354, 0.90699367,\n",
       "        0.91962025, 0.91202532, 0.92962025, 0.92218354, 0.9296519 ,\n",
       "        0.90949367, 0.93468354, 0.92468354, 0.93468354, 0.90452532,\n",
       "        0.93218354, 0.89943038, 0.93721519, 0.92718354, 0.93468354,\n",
       "        0.92721519, 0.93468354, 0.92721519, 0.92968354, 0.90199367,\n",
       "        0.92962025, 0.92218354, 0.9296519 , 0.90949367, 0.93468354,\n",
       "        0.92468354, 0.93468354, 0.90452532, 0.93218354, 0.89943038,\n",
       "        0.93721519, 0.92718354, 0.93468354, 0.92721519, 0.93468354,\n",
       "        0.92721519, 0.92968354, 0.90199367, 0.92962025, 0.92218354,\n",
       "        0.9296519 , 0.90949367, 0.93468354, 0.92468354, 0.93468354,\n",
       "        0.90452532, 0.93218354, 0.89943038, 0.93721519, 0.92718354,\n",
       "        0.93468354, 0.92721519, 0.93468354, 0.92721519, 0.92968354,\n",
       "        0.90199367, 0.92962025, 0.92218354, 0.9296519 , 0.90949367,\n",
       "        0.93468354, 0.92468354, 0.93468354, 0.90452532, 0.93218354,\n",
       "        0.89943038, 0.93721519, 0.92718354, 0.93468354, 0.92721519,\n",
       "        0.93468354, 0.92721519, 0.92968354, 0.90199367, 0.92962025,\n",
       "        0.92218354, 0.9296519 , 0.90949367, 0.93468354, 0.92468354,\n",
       "        0.93468354, 0.90452532, 0.93218354, 0.89943038, 0.93721519,\n",
       "        0.92718354, 0.93468354, 0.92721519, 0.93468354, 0.92721519,\n",
       "        0.92968354, 0.90199367, 0.92962025, 0.92218354, 0.9296519 ,\n",
       "        0.90949367, 0.93468354, 0.92468354, 0.93468354, 0.90452532,\n",
       "        0.93218354, 0.89943038, 0.93721519, 0.92718354, 0.93468354,\n",
       "        0.92721519, 0.93468354, 0.92721519, 0.92968354, 0.90199367,\n",
       "        0.92962025, 0.92218354, 0.9296519 , 0.90949367, 0.93468354,\n",
       "        0.92468354, 0.93468354, 0.90452532, 0.93218354, 0.89943038,\n",
       "        0.93721519, 0.92718354, 0.93468354, 0.92721519, 0.93468354,\n",
       "        0.92721519, 0.92968354, 0.90199367, 0.92962025, 0.92218354,\n",
       "        0.9296519 , 0.90949367, 0.93468354, 0.92468354, 0.93468354,\n",
       "        0.90452532, 0.93218354, 0.89943038, 0.93721519, 0.92718354,\n",
       "        0.93468354, 0.92721519, 0.93468354, 0.92721519, 0.92968354,\n",
       "        0.90199367]),\n",
       " 'std_test_score': array([0.0162916 , 0.02328379, 0.01841397, 0.02801421, 0.01694472,\n",
       "        0.02168878, 0.02053478, 0.03434843, 0.0144192 , 0.0226135 ,\n",
       "        0.0126063 , 0.02440089, 0.01495252, 0.03255473, 0.01495252,\n",
       "        0.03255473, 0.01684989, 0.03742696, 0.0162916 , 0.02328379,\n",
       "        0.01841397, 0.02801421, 0.01694472, 0.02168878, 0.02053478,\n",
       "        0.03434843, 0.0144192 , 0.0226135 , 0.0126063 , 0.02440089,\n",
       "        0.01495252, 0.03255473, 0.01495252, 0.03255473, 0.01684989,\n",
       "        0.03742696, 0.0162916 , 0.02328379, 0.01841397, 0.02801421,\n",
       "        0.01694472, 0.02168878, 0.02053478, 0.03434843, 0.0144192 ,\n",
       "        0.0226135 , 0.0126063 , 0.02440089, 0.01495252, 0.03255473,\n",
       "        0.01495252, 0.03255473, 0.01684989, 0.03742696, 0.0162916 ,\n",
       "        0.02328379, 0.01841397, 0.02801421, 0.01694472, 0.02168878,\n",
       "        0.02053478, 0.03434843, 0.0144192 , 0.0226135 , 0.0126063 ,\n",
       "        0.02440089, 0.01495252, 0.03255473, 0.01495252, 0.03255473,\n",
       "        0.01684989, 0.03742696, 0.02044764, 0.02417697, 0.02317778,\n",
       "        0.01865506, 0.02894236, 0.01916348, 0.01833216, 0.02709434,\n",
       "        0.02311895, 0.01813338, 0.028451  , 0.02136167, 0.0266957 ,\n",
       "        0.03088306, 0.0266957 , 0.03088306, 0.02568065, 0.02923605,\n",
       "        0.02044764, 0.02417697, 0.02317778, 0.01865506, 0.02894236,\n",
       "        0.01916348, 0.01833216, 0.02709434, 0.02311895, 0.01813338,\n",
       "        0.028451  , 0.02136167, 0.0266957 , 0.03088306, 0.0266957 ,\n",
       "        0.03088306, 0.02568065, 0.02923605, 0.02044764, 0.02417697,\n",
       "        0.02317778, 0.01865506, 0.02894236, 0.01916348, 0.01833216,\n",
       "        0.02709434, 0.02311895, 0.01813338, 0.028451  , 0.02136167,\n",
       "        0.0266957 , 0.03088306, 0.0266957 , 0.03088306, 0.02568065,\n",
       "        0.02923605, 0.02044764, 0.02417697, 0.02317778, 0.01865506,\n",
       "        0.02894236, 0.01916348, 0.01833216, 0.02709434, 0.02311895,\n",
       "        0.01813338, 0.028451  , 0.02136167, 0.0266957 , 0.03088306,\n",
       "        0.0266957 , 0.03088306, 0.02568065, 0.02923605, 0.02044764,\n",
       "        0.02417697, 0.02317778, 0.01865506, 0.02894236, 0.01916348,\n",
       "        0.01833216, 0.02709434, 0.02311895, 0.01813338, 0.028451  ,\n",
       "        0.02136167, 0.0266957 , 0.03088306, 0.0266957 , 0.03088306,\n",
       "        0.02568065, 0.02923605, 0.02044764, 0.02417697, 0.02317778,\n",
       "        0.01865506, 0.02894236, 0.01916348, 0.01833216, 0.02709434,\n",
       "        0.02311895, 0.01813338, 0.028451  , 0.02136167, 0.0266957 ,\n",
       "        0.03088306, 0.0266957 , 0.03088306, 0.02568065, 0.02923605,\n",
       "        0.02044764, 0.02417697, 0.02317778, 0.01865506, 0.02894236,\n",
       "        0.01916348, 0.01833216, 0.02709434, 0.02311895, 0.01813338,\n",
       "        0.028451  , 0.02136167, 0.0266957 , 0.03088306, 0.0266957 ,\n",
       "        0.03088306, 0.02568065, 0.02923605, 0.02044764, 0.02417697,\n",
       "        0.02317778, 0.01865506, 0.02894236, 0.01916348, 0.01833216,\n",
       "        0.02709434, 0.02311895, 0.01813338, 0.028451  , 0.02136167,\n",
       "        0.0266957 , 0.03088306, 0.0266957 , 0.03088306, 0.02568065,\n",
       "        0.02923605]),\n",
       " 'rank_test_score': array([173,  61, 145,  41, 157,  65, 157, 165, 113, 133, 149,  65,  53,\n",
       "        185,  53, 185, 149, 169, 173,  61, 145,  41, 157,  65, 157, 165,\n",
       "        113, 133, 149,  65,  53, 185,  53, 185, 149, 169, 173,  61, 145,\n",
       "         41, 157,  65, 157, 165, 113, 133, 149,  65,  53, 185,  53, 185,\n",
       "        149, 169, 173,  61, 145,  41, 157,  65, 157, 165, 113, 133, 149,\n",
       "         65,  53, 185,  53, 185, 149, 169,  89, 137,  81, 177,   9, 125,\n",
       "          9, 193,  45, 209,   1, 113,   9,  97,   9,  97,  65, 201,  89,\n",
       "        137,  81, 177,   9, 125,   9, 193,  45, 209,   1, 113,   9,  97,\n",
       "          9,  97,  65, 201,  89, 137,  81, 177,   9, 125,   9, 193,  45,\n",
       "        209,   1, 113,   9,  97,   9,  97,  65, 201,  89, 137,  81, 177,\n",
       "          9, 125,   9, 193,  45, 209,   1, 113,   9,  97,   9,  97,  65,\n",
       "        201,  89, 137,  81, 177,   9, 125,   9, 193,  45, 209,   1, 113,\n",
       "          9,  97,   9,  97,  65, 201,  89, 137,  81, 177,   9, 125,   9,\n",
       "        193,  45, 209,   1, 113,   9,  97,   9,  97,  65, 201,  89, 137,\n",
       "         81, 177,   9, 125,   9, 193,  45, 209,   1, 113,   9,  97,   9,\n",
       "         97,  65, 201,  89, 137,  81, 177,   9, 125,   9, 193,  45, 209,\n",
       "          1, 113,   9,  97,   9,  97,  65, 201])}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_grid_search.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c15fe7d8-fdbd-47a0-ac06-aed7ce98f110",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_criterion</th>\n",
       "      <th>param_max_depth</th>\n",
       "      <th>param_min_samples_leaf</th>\n",
       "      <th>param_min_samples_split</th>\n",
       "      <th>param_splitter</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.018306</td>\n",
       "      <td>0.002134</td>\n",
       "      <td>0.004440</td>\n",
       "      <td>0.002862</td>\n",
       "      <td>gini</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>best</td>\n",
       "      <td>{'criterion': 'gini', 'max_depth': None, 'min_...</td>\n",
       "      <td>0.9125</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9000</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.911392</td>\n",
       "      <td>0.909620</td>\n",
       "      <td>0.016292</td>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006251</td>\n",
       "      <td>0.007656</td>\n",
       "      <td>gini</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>random</td>\n",
       "      <td>{'criterion': 'gini', 'max_depth': None, 'min_...</td>\n",
       "      <td>0.9375</td>\n",
       "      <td>0.9000</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.911392</td>\n",
       "      <td>0.949367</td>\n",
       "      <td>0.932152</td>\n",
       "      <td>0.023284</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.015628</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.006251</td>\n",
       "      <td>0.007655</td>\n",
       "      <td>gini</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>best</td>\n",
       "      <td>{'criterion': 'gini', 'max_depth': None, 'min_...</td>\n",
       "      <td>0.9125</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9250</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.919684</td>\n",
       "      <td>0.018414</td>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006249</td>\n",
       "      <td>0.007653</td>\n",
       "      <td>gini</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>random</td>\n",
       "      <td>{'criterion': 'gini', 'max_depth': None, 'min_...</td>\n",
       "      <td>0.9250</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.962025</td>\n",
       "      <td>0.932215</td>\n",
       "      <td>0.028014</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.018750</td>\n",
       "      <td>0.006251</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>gini</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>best</td>\n",
       "      <td>{'criterion': 'gini', 'max_depth': None, 'min_...</td>\n",
       "      <td>0.9375</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9250</td>\n",
       "      <td>0.911392</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.917089</td>\n",
       "      <td>0.016945</td>\n",
       "      <td>157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.007654</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>log_loss</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>random</td>\n",
       "      <td>{'criterion': 'log_loss', 'max_depth': 30, 'mi...</td>\n",
       "      <td>0.9125</td>\n",
       "      <td>0.8750</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.949367</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.927215</td>\n",
       "      <td>0.030883</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>0.012500</td>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.006250</td>\n",
       "      <td>0.007655</td>\n",
       "      <td>log_loss</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>best</td>\n",
       "      <td>{'criterion': 'log_loss', 'max_depth': 30, 'mi...</td>\n",
       "      <td>0.9500</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.949367</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.934684</td>\n",
       "      <td>0.026696</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006249</td>\n",
       "      <td>0.007654</td>\n",
       "      <td>log_loss</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>random</td>\n",
       "      <td>{'criterion': 'log_loss', 'max_depth': 30, 'mi...</td>\n",
       "      <td>0.9125</td>\n",
       "      <td>0.8750</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.949367</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.927215</td>\n",
       "      <td>0.030883</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>0.014415</td>\n",
       "      <td>0.008105</td>\n",
       "      <td>0.003926</td>\n",
       "      <td>0.006052</td>\n",
       "      <td>log_loss</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>best</td>\n",
       "      <td>{'criterion': 'log_loss', 'max_depth': 30, 'mi...</td>\n",
       "      <td>0.9250</td>\n",
       "      <td>0.8875</td>\n",
       "      <td>0.9625</td>\n",
       "      <td>0.949367</td>\n",
       "      <td>0.924051</td>\n",
       "      <td>0.929684</td>\n",
       "      <td>0.025681</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>0.000800</td>\n",
       "      <td>0.001600</td>\n",
       "      <td>0.007195</td>\n",
       "      <td>0.007103</td>\n",
       "      <td>log_loss</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>random</td>\n",
       "      <td>{'criterion': 'log_loss', 'max_depth': 30, 'mi...</td>\n",
       "      <td>0.9125</td>\n",
       "      <td>0.8750</td>\n",
       "      <td>0.9250</td>\n",
       "      <td>0.936709</td>\n",
       "      <td>0.860759</td>\n",
       "      <td>0.901994</td>\n",
       "      <td>0.029236</td>\n",
       "      <td>201</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>216 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     mean_fit_time  std_fit_time  mean_score_time  std_score_time  \\\n",
       "0         0.018306      0.002134         0.004440        0.002862   \n",
       "1         0.000000      0.000000         0.006251        0.007656   \n",
       "2         0.015628      0.000005         0.006251        0.007655   \n",
       "3         0.000000      0.000000         0.006249        0.007653   \n",
       "4         0.018750      0.006251         0.000000        0.000000   \n",
       "..             ...           ...              ...             ...   \n",
       "211       0.006250      0.007654         0.000000        0.000000   \n",
       "212       0.012500      0.006250         0.006250        0.007655   \n",
       "213       0.000000      0.000000         0.006249        0.007654   \n",
       "214       0.014415      0.008105         0.003926        0.006052   \n",
       "215       0.000800      0.001600         0.007195        0.007103   \n",
       "\n",
       "    param_criterion param_max_depth param_min_samples_leaf  \\\n",
       "0              gini            None                      1   \n",
       "1              gini            None                      1   \n",
       "2              gini            None                      1   \n",
       "3              gini            None                      1   \n",
       "4              gini            None                      1   \n",
       "..              ...             ...                    ...   \n",
       "211        log_loss              30                      4   \n",
       "212        log_loss              30                      4   \n",
       "213        log_loss              30                      4   \n",
       "214        log_loss              30                      4   \n",
       "215        log_loss              30                      4   \n",
       "\n",
       "    param_min_samples_split param_splitter  \\\n",
       "0                         2           best   \n",
       "1                         2         random   \n",
       "2                         5           best   \n",
       "3                         5         random   \n",
       "4                        10           best   \n",
       "..                      ...            ...   \n",
       "211                       2         random   \n",
       "212                       5           best   \n",
       "213                       5         random   \n",
       "214                      10           best   \n",
       "215                      10         random   \n",
       "\n",
       "                                                params  split0_test_score  \\\n",
       "0    {'criterion': 'gini', 'max_depth': None, 'min_...             0.9125   \n",
       "1    {'criterion': 'gini', 'max_depth': None, 'min_...             0.9375   \n",
       "2    {'criterion': 'gini', 'max_depth': None, 'min_...             0.9125   \n",
       "3    {'criterion': 'gini', 'max_depth': None, 'min_...             0.9250   \n",
       "4    {'criterion': 'gini', 'max_depth': None, 'min_...             0.9375   \n",
       "..                                                 ...                ...   \n",
       "211  {'criterion': 'log_loss', 'max_depth': 30, 'mi...             0.9125   \n",
       "212  {'criterion': 'log_loss', 'max_depth': 30, 'mi...             0.9500   \n",
       "213  {'criterion': 'log_loss', 'max_depth': 30, 'mi...             0.9125   \n",
       "214  {'criterion': 'log_loss', 'max_depth': 30, 'mi...             0.9250   \n",
       "215  {'criterion': 'log_loss', 'max_depth': 30, 'mi...             0.9125   \n",
       "\n",
       "     split1_test_score  split2_test_score  split3_test_score  \\\n",
       "0               0.8875             0.9000           0.936709   \n",
       "1               0.9000             0.9625           0.911392   \n",
       "2               0.8875             0.9250           0.936709   \n",
       "3               0.8875             0.9625           0.924051   \n",
       "4               0.8875             0.9250           0.911392   \n",
       "..                 ...                ...                ...   \n",
       "211             0.8750             0.9625           0.949367   \n",
       "212             0.8875             0.9625           0.949367   \n",
       "213             0.8750             0.9625           0.949367   \n",
       "214             0.8875             0.9625           0.949367   \n",
       "215             0.8750             0.9250           0.936709   \n",
       "\n",
       "     split4_test_score  mean_test_score  std_test_score  rank_test_score  \n",
       "0             0.911392         0.909620        0.016292              173  \n",
       "1             0.949367         0.932152        0.023284               61  \n",
       "2             0.936709         0.919684        0.018414              145  \n",
       "3             0.962025         0.932215        0.028014               41  \n",
       "4             0.924051         0.917089        0.016945              157  \n",
       "..                 ...              ...             ...              ...  \n",
       "211           0.936709         0.927215        0.030883               97  \n",
       "212           0.924051         0.934684        0.026696                9  \n",
       "213           0.936709         0.927215        0.030883               97  \n",
       "214           0.924051         0.929684        0.025681               65  \n",
       "215           0.860759         0.901994        0.029236              201  \n",
       "\n",
       "[216 rows x 18 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuning_result_dt_gs = pd.DataFrame(dt_grid_search.cv_results_)\n",
    "tuning_result_dt_gs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e282d562-0d69-424a-a68c-5b0f4abbf4fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'criterion': 'entropy',\n",
       " 'max_depth': None,\n",
       " 'min_samples_leaf': 2,\n",
       " 'min_samples_split': 10,\n",
       " 'splitter': 'best'}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_grid_search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f6cba26-0557-4dd3-931c-a2faec4ccd3b",
   "metadata": {},
   "source": [
    "**Evaluate Decision Tree Optimized model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c5511f77-d57f-4869-8485-8686f32d4ba6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Decision Tree Grid Search Performance:\n",
      "Accuracy: 0.9590643274853801\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           B       0.95      0.98      0.97       108\n",
      "           M       0.97      0.92      0.94        63\n",
      "\n",
      "    accuracy                           0.96       171\n",
      "   macro avg       0.96      0.95      0.96       171\n",
      "weighted avg       0.96      0.96      0.96       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Decision Tree and Grid Search\n",
    "y_pred_dt_gs = dt_grid_search.predict(xtest)\n",
    "print(\"\\nDecision Tree Grid Search Performance:\")\n",
    "print(\"Accuracy:\", accuracy_score(ytest, y_pred_dt_gs))\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(ytest, y_pred_dt_gs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12e9a5ce-3bb6-4546-bde7-9fee19b7c6f4",
   "metadata": {},
   "source": [
    "**Grid Search for Random Forest**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a54edd75-1f1a-4536-a8a0-f05bf4c40a0b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "             param_grid={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;, &#x27;log_loss&#x27;],\n",
       "                         &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                         &#x27;max_features&#x27;: [None, &#x27;sqrt&#x27;, &#x27;log2&#x27;],\n",
       "                         &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                         &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-4\" type=\"checkbox\" ><label for=\"sk-estimator-id-4\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "             param_grid={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;, &#x27;log_loss&#x27;],\n",
       "                         &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                         &#x27;max_features&#x27;: [None, &#x27;sqrt&#x27;, &#x27;log2&#x27;],\n",
       "                         &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                         &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-5\" type=\"checkbox\" ><label for=\"sk-estimator-id-5\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-6\" type=\"checkbox\" ><label for=\"sk-estimator-id-6\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "             param_grid={'criterion': ['gini', 'entropy', 'log_loss'],\n",
       "                         'max_depth': [None, 10, 20, 30],\n",
       "                         'max_features': [None, 'sqrt', 'log2'],\n",
       "                         'min_samples_leaf': [1, 2, 4],\n",
       "                         'min_samples_split': [2, 5, 10],\n",
       "                         'n_estimators': [100, 200, 300]})"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid_search = GridSearchCV(estimator = RandomForestClassifier(random_state = 42),\n",
    "                             param_grid = rf_params, cv = 5)\n",
    "rf_grid_search.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4c37518c-7fe0-4015-99a1-3cae0551f751",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'criterion': 'entropy',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'sqrt',\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'n_estimators': 100}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_grid_search.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f26f134f-bf74-4b51-8014-1557bf17ceac",
   "metadata": {},
   "source": [
    "**Evaluate Random Forest Grid Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "91917f65-bdcc-442c-80e3-b0d5d50e88a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Decision Tree Grid Search Performance:\n",
      "Accuracy: 0.9707602339181286\n",
      "Classification Report:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           B       0.96      0.99      0.98       108\n",
      "           M       0.98      0.94      0.96        63\n",
      "\n",
      "    accuracy                           0.97       171\n",
      "   macro avg       0.97      0.96      0.97       171\n",
      "weighted avg       0.97      0.97      0.97       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Random Forest and Grid Search\n",
    "y_pred_rf = rf_grid_search.predict(xtest)\n",
    "print(\"\\nDecision Tree Grid Search Performance:\")\n",
    "print(\"Accuracy:\", accuracy_score(ytest, y_pred_rf))\n",
    "print(\"Classification Report:\")\n",
    "print(classification_report(ytest, y_pred_rf))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1ad166b-c684-4469-b675-b663c5ec2c2b",
   "metadata": {},
   "source": [
    "# Perform and Evaluation Randomized Search"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6f46921-f9e8-4b9f-8fac-cea397a392f6",
   "metadata": {},
   "source": [
    "**Randomized Search for Decision tree**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "59e05fa8-cf8a-4b21-b9dd-06231401e035",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;,\n",
       "                                                      &#x27;log_loss&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;splitter&#x27;: [&#x27;best&#x27;, &#x27;random&#x27;]},\n",
       "                   random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;,\n",
       "                                                      &#x27;log_loss&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;splitter&#x27;: [&#x27;best&#x27;, &#x27;random&#x27;]},\n",
       "                   random_state=42)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">DecisionTreeClassifier</label><div class=\"sk-toggleable__content\"><pre>DecisionTreeClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=DecisionTreeClassifier(random_state=42),\n",
       "                   param_distributions={'criterion': ['gini', 'entropy',\n",
       "                                                      'log_loss'],\n",
       "                                        'max_depth': [None, 10, 20, 30],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'splitter': ['best', 'random']},\n",
       "                   random_state=42)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# perfrom randomized search for decision tree\n",
    "'''''\n",
    "class sklearn.model_selection.RandomizedSearchCV\n",
    "(estimator, param_distributions, *, n_iter=10, \n",
    "scoring=None, n_jobs=None, refit=True, cv=None,\n",
    "verbose=0, pre_dispatch='2*n_jobs', random_state=None, \n",
    "error_score=nan, return_train_score=False)\n",
    "'''''\n",
    "dt_rs = RandomizedSearchCV(estimator = DecisionTreeClassifier(random_state = 42), \n",
    "                           param_distributions = dt_params, n_iter = 10, cv = 5, random_state = 42)\n",
    "dt_rs.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "aa575804-b3b6-49dd-a93c-6db0d8a1540d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00390191, 0.00313148, 0.01561828, 0.        , 0.00937505,\n",
       "        0.00312552, 0.01562481, 0.00312524, 0.00312486, 0.0062501 ]),\n",
       " 'std_fit_time': array([6.66683583e-04, 6.26296997e-03, 1.38096849e-05, 0.00000000e+00,\n",
       "        7.65469440e-03, 6.25104904e-03, 7.44843452e-07, 6.25047684e-03,\n",
       "        6.24971390e-03, 7.65477225e-03]),\n",
       " 'mean_score_time': array([0.00454192, 0.00625148, 0.        , 0.00625076, 0.        ,\n",
       "        0.00625029, 0.00312505, 0.003125  , 0.00624852, 0.003125  ]),\n",
       " 'std_score_time': array([0.00387319, 0.00765647, 0.        , 0.00765559, 0.        ,\n",
       "        0.00765501, 0.0062501 , 0.00625   , 0.00765285, 0.00625   ]),\n",
       " 'param_splitter': masked_array(data=['random', 'random', 'best', 'random', 'random',\n",
       "                    'random', 'best', 'random', 'random', 'random'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_samples_split': masked_array(data=[5, 5, 2, 5, 5, 5, 5, 2, 2, 2],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_samples_leaf': masked_array(data=[1, 4, 4, 4, 4, 1, 1, 1, 2, 4],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_max_depth': masked_array(data=[30, 30, 30, 10, None, 20, 20, None, 30, 30],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_criterion': masked_array(data=['log_loss', 'log_loss', 'entropy', 'log_loss', 'gini',\n",
       "                    'entropy', 'log_loss', 'entropy', 'log_loss',\n",
       "                    'entropy'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'splitter': 'random',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 1,\n",
       "   'max_depth': 30,\n",
       "   'criterion': 'log_loss'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 4,\n",
       "   'max_depth': 30,\n",
       "   'criterion': 'log_loss'},\n",
       "  {'splitter': 'best',\n",
       "   'min_samples_split': 2,\n",
       "   'min_samples_leaf': 4,\n",
       "   'max_depth': 30,\n",
       "   'criterion': 'entropy'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 4,\n",
       "   'max_depth': 10,\n",
       "   'criterion': 'log_loss'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 4,\n",
       "   'max_depth': None,\n",
       "   'criterion': 'gini'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 1,\n",
       "   'max_depth': 20,\n",
       "   'criterion': 'entropy'},\n",
       "  {'splitter': 'best',\n",
       "   'min_samples_split': 5,\n",
       "   'min_samples_leaf': 1,\n",
       "   'max_depth': 20,\n",
       "   'criterion': 'log_loss'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 2,\n",
       "   'min_samples_leaf': 1,\n",
       "   'max_depth': None,\n",
       "   'criterion': 'entropy'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 2,\n",
       "   'min_samples_leaf': 2,\n",
       "   'max_depth': 30,\n",
       "   'criterion': 'log_loss'},\n",
       "  {'splitter': 'random',\n",
       "   'min_samples_split': 2,\n",
       "   'min_samples_leaf': 4,\n",
       "   'max_depth': 30,\n",
       "   'criterion': 'entropy'}],\n",
       " 'split0_test_score': array([0.925 , 0.9125, 0.95  , 0.9125, 0.925 , 0.925 , 0.95  , 0.925 ,\n",
       "        0.9375, 0.9125]),\n",
       " 'split1_test_score': array([0.8875, 0.875 , 0.8875, 0.875 , 0.875 , 0.8875, 0.8875, 0.8875,\n",
       "        0.8875, 0.875 ]),\n",
       " 'split2_test_score': array([0.9375, 0.9625, 0.9625, 0.9625, 0.9375, 0.9375, 0.95  , 0.925 ,\n",
       "        0.8875, 0.9625]),\n",
       " 'split3_test_score': array([0.89873418, 0.94936709, 0.94936709, 0.94936709, 0.93670886,\n",
       "        0.89873418, 0.93670886, 0.96202532, 0.93670886, 0.94936709]),\n",
       " 'split4_test_score': array([0.89873418, 0.93670886, 0.92405063, 0.93670886, 0.86075949,\n",
       "        0.89873418, 0.92405063, 0.91139241, 0.87341772, 0.93670886]),\n",
       " 'mean_test_score': array([0.90949367, 0.92721519, 0.93468354, 0.92721519, 0.90699367,\n",
       "        0.90949367, 0.9296519 , 0.92218354, 0.90452532, 0.92721519]),\n",
       " 'std_test_score': array([0.01865506, 0.03088306, 0.0266957 , 0.03088306, 0.03255473,\n",
       "        0.01865506, 0.02317778, 0.02417697, 0.02709434, 0.03088306]),\n",
       " 'rank_test_score': array([ 7,  3,  1,  3,  9,  7,  2,  6, 10,  3])}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_rs.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "51615cb8-b176-4065-b207-bdfb1fa948af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'splitter': 'best',\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 4,\n",
       " 'max_depth': 30,\n",
       " 'criterion': 'entropy'}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dt_rs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4e00346-b1a3-4578-a3c7-d41757697f07",
   "metadata": {},
   "source": [
    "**Perforfance Evaluation Decision tree for Randomized Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d0eaabea-a3df-41a5-b5c5-fd6fc92a7356",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Randomized Search Performance:\n",
      "Accuracy 0.9590643274853801\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.95      0.99      0.97       108\n",
      "           M       0.98      0.90      0.94        63\n",
      "\n",
      "    accuracy                           0.96       171\n",
      "   macro avg       0.96      0.95      0.96       171\n",
      "weighted avg       0.96      0.96      0.96       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred_dt_rs = dt_rs.predict(xtest)\n",
    "print('Decision Tree Randomized Search Performance:')\n",
    "print('Accuracy', accuracy_score(ytest,y_pred_dt_rs))\n",
    "print('Classification Report:\\n',classification_report(ytest,y_pred_dt_rs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "921b681f-aff8-4c2c-90d1-bec5b29ed9af",
   "metadata": {},
   "source": [
    "**Randomized Search for Random Forest**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8fa95f65-52c5-4214-bc2e-a066f7f30aa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {color: black;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;,\n",
       "                                                      &#x27;log_loss&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                                        &#x27;max_features&#x27;: [None, &#x27;sqrt&#x27;, &#x27;log2&#x27;],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300]},\n",
       "                   random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;criterion&#x27;: [&#x27;gini&#x27;, &#x27;entropy&#x27;,\n",
       "                                                      &#x27;log_loss&#x27;],\n",
       "                                        &#x27;max_depth&#x27;: [None, 10, 20, 30],\n",
       "                                        &#x27;max_features&#x27;: [None, &#x27;sqrt&#x27;, &#x27;log2&#x27;],\n",
       "                                        &#x27;min_samples_leaf&#x27;: [1, 2, 4],\n",
       "                                        &#x27;min_samples_split&#x27;: [2, 5, 10],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300]},\n",
       "                   random_state=42)</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=RandomForestClassifier(random_state=42),\n",
       "                   param_distributions={'criterion': ['gini', 'entropy',\n",
       "                                                      'log_loss'],\n",
       "                                        'max_depth': [None, 10, 20, 30],\n",
       "                                        'max_features': [None, 'sqrt', 'log2'],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'n_estimators': [100, 200, 300]},\n",
       "                   random_state=42)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_rs = RandomizedSearchCV(estimator = RandomForestClassifier(random_state = 42),\n",
    "                          param_distributions = rf_params, cv = 5, random_state = 42)\n",
    "rf_rs.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2cf7dbed-5218-4714-97db-526b639002fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 200,\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 2,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': 30,\n",
       " 'criterion': 'gini'}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_rs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "220d317b-3bd3-4d02-b40b-2550466f0f1b",
   "metadata": {},
   "source": [
    "**Perforfance Evaluation Random Forest for Randomized Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c8724ce9-9efd-4b05-889f-1ba80d15768f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Randomized Search Performance:\n",
      "Accuracy 0.9707602339181286\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.96      0.99      0.98       108\n",
      "           M       0.98      0.94      0.96        63\n",
      "\n",
      "    accuracy                           0.97       171\n",
      "   macro avg       0.97      0.96      0.97       171\n",
      "weighted avg       0.97      0.97      0.97       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred_rf_rs = rf_rs.predict(xtest)\n",
    "print('Decision Tree Randomized Search Performance:')\n",
    "print('Accuracy', accuracy_score(ytest,y_pred_rf_rs))\n",
    "print('Classification Report:\\n',classification_report(ytest,y_pred_rf_rs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c980e532-348d-4ed4-ab14-5b2989b024ff",
   "metadata": {},
   "source": [
    "# AdaBoost Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5b876639-6b22-4cae-aa44-f7338f6e716d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ddbe214-9e32-4d63-8232-8792b44e0994",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''''\n",
    "class sklearn.ensemble.AdaBoostClassifier(\n",
    "    estimator=None, *, n_estimators=50, learning_rate=1.0, \n",
    "    algorithm='SAMME.R', random_state=None, \n",
    "    base_estimator='deprecated')\n",
    "    '''''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23533731-966d-4f81-9155-7fed53b81ccb",
   "metadata": {},
   "source": [
    "**AdaBoost Classifier**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "aa1a6d5a-4565-4a65-8415-e110048877bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>AdaBoostClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-20\" type=\"checkbox\" checked><label for=\"sk-estimator-id-20\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AdaBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>AdaBoostClassifier(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "AdaBoostClassifier(random_state=42)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "adaBoost = AdaBoostClassifier(random_state =42)\n",
    "adaBoost.fit(xtrain,ytrain)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b8dec85a-52b4-497f-9d78-69655cab2873",
   "metadata": {},
   "outputs": [],
   "source": [
    "adaBoost_ypred = adaBoost.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ac17923d-fffb-4bb4-8eed-80deef9583c6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Performance (estimator = Decision Tree):\n",
      "\n",
      "Accuracy 0.9766081871345029\n"
     ]
    }
   ],
   "source": [
    "print('AdaBoost Performance (estimator = Decision Tree):\\n',)\n",
    "print('Accuracy', accuracy_score(ytest,adaBoost_ypred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f131fe06-e0f8-4f2f-bc4f-9e922b63749f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.98      0.98      0.98       108\n",
      "           M       0.97      0.97      0.97        63\n",
      "\n",
      "    accuracy                           0.98       171\n",
      "   macro avg       0.97      0.97      0.97       171\n",
      "weighted avg       0.98      0.98      0.98       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification Report:\\n\\n',classification_report(ytest,adaBoost_ypred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edcccfbe-9b90-4449-84bc-b6aded879380",
   "metadata": {},
   "source": [
    "# AdaBoost Hyperparameter Optimization and Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b91d0842-6b30-45ec-9547-a5ab079a4f26",
   "metadata": {},
   "source": [
    "**ADA Boost optimization using GRID search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ce36cb1-9172-4292-9fee-c5c2fb128a88",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''''\n",
    "class sklearn.ensemble.AdaBoostClassifier(\n",
    "    estimator=None, *, n_estimators=50, learning_rate=1.0, \n",
    "    algorithm='SAMME.R', random_state=None, \n",
    "    base_estimator='deprecated')\n",
    "    '''''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7518b6e1-bfb8-43bd-9b76-4e72b7d4a2aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_params = {\n",
    "    'n_estimators' : [100,200,300,400],\n",
    "    'learning_rate' : [0.0001, 0.001, 0.1, 1.0],\n",
    "    'algorithm' : ['SAMME', 'SAMME.R']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "fe98dce0-3510-4f86-aabf-92d31cf341d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-5 {color: black;}#sk-container-id-5 pre{padding: 0;}#sk-container-id-5 div.sk-toggleable {background-color: white;}#sk-container-id-5 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-5 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-5 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-5 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-5 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-5 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-5 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-5 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-5 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-5 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-5 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-5 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-5 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-5 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-5 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-5 div.sk-item {position: relative;z-index: 1;}#sk-container-id-5 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-5 div.sk-item::before, #sk-container-id-5 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-5 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-5 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-5 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-5 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-5 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-5 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-5 div.sk-label-container {text-align: center;}#sk-container-id-5 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-5 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-5\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "             param_grid={&#x27;algorithm&#x27;: [&#x27;SAMME&#x27;, &#x27;SAMME.R&#x27;],\n",
       "                         &#x27;learning_rate&#x27;: [0.0001, 0.001, 0.1, 1.0],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300, 400]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-14\" type=\"checkbox\" ><label for=\"sk-estimator-id-14\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "             param_grid={&#x27;algorithm&#x27;: [&#x27;SAMME&#x27;, &#x27;SAMME.R&#x27;],\n",
       "                         &#x27;learning_rate&#x27;: [0.0001, 0.001, 0.1, 1.0],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300, 400]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-15\" type=\"checkbox\" ><label for=\"sk-estimator-id-15\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: AdaBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>AdaBoostClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-16\" type=\"checkbox\" ><label for=\"sk-estimator-id-16\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AdaBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>AdaBoostClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "             param_grid={'algorithm': ['SAMME', 'SAMME.R'],\n",
       "                         'learning_rate': [0.0001, 0.001, 0.1, 1.0],\n",
       "                         'n_estimators': [100, 200, 300, 400]})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AdaBoost = AdaBoostClassifier(random_state = 42)\n",
    "AdaBoost_gs = GridSearchCV(estimator = AdaBoost,param_grid = ada_params, cv = 5)\n",
    "AdaBoost_gs.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "da32d6b6-a04b-4d1b-be93-28258a0e9884",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'algorithm': 'SAMME', 'learning_rate': 1.0, 'n_estimators': 200}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AdaBoost_gs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d537d54-3299-4c5a-aebb-3a1202b46e7f",
   "metadata": {},
   "source": [
    "**AdaBoost Grid search Performance Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ea3b844a-1d2b-4fb7-87e7-0bb4764b1c97",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Grid Search Accuracy:\n",
      "\n",
      " 0.9824561403508771\n"
     ]
    }
   ],
   "source": [
    "ypred_AdaBoost_gs = AdaBoost_gs.predict(xtest)\n",
    "print('AdaBoost Grid Search Accuracy:\\n\\n', accuracy_score(ytest,ypred_AdaBoost_gs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "526ba7e1-c963-44a3-8c29-3712471f059c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Grid Search Classification Report:\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.99      0.98      0.99       108\n",
      "           M       0.97      0.98      0.98        63\n",
      "\n",
      "    accuracy                           0.98       171\n",
      "   macro avg       0.98      0.98      0.98       171\n",
      "weighted avg       0.98      0.98      0.98       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('AdaBoost Grid Search Classification Report:\\n\\n', classification_report(ytest,ypred_AdaBoost_gs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a52e2a0-567b-45de-aeea-b249a45e091d",
   "metadata": {},
   "source": [
    "**ADA Boost optimization using RANDOMIZED search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "58f57072-2428-4986-88d3-851a4ab41dad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-6 {color: black;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;algorithm&#x27;: [&#x27;SAMME&#x27;, &#x27;SAMME.R&#x27;],\n",
       "                                        &#x27;learning_rate&#x27;: [0.0001, 0.001, 0.1,\n",
       "                                                          1.0],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300, 400]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-17\" type=\"checkbox\" ><label for=\"sk-estimator-id-17\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "                   param_distributions={&#x27;algorithm&#x27;: [&#x27;SAMME&#x27;, &#x27;SAMME.R&#x27;],\n",
       "                                        &#x27;learning_rate&#x27;: [0.0001, 0.001, 0.1,\n",
       "                                                          1.0],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300, 400]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-18\" type=\"checkbox\" ><label for=\"sk-estimator-id-18\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: AdaBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>AdaBoostClassifier(random_state=42)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-19\" type=\"checkbox\" ><label for=\"sk-estimator-id-19\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">AdaBoostClassifier</label><div class=\"sk-toggleable__content\"><pre>AdaBoostClassifier(random_state=42)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=5, estimator=AdaBoostClassifier(random_state=42),\n",
       "                   param_distributions={'algorithm': ['SAMME', 'SAMME.R'],\n",
       "                                        'learning_rate': [0.0001, 0.001, 0.1,\n",
       "                                                          1.0],\n",
       "                                        'n_estimators': [100, 200, 300, 400]})"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Ada_Boost = AdaBoostClassifier(random_state = 42)\n",
    "AdaBoost_rs = RandomizedSearchCV(estimator = Ada_Boost, param_distributions = ada_params, \n",
    "                                 n_iter = 10, cv = 5)\n",
    "AdaBoost_rs.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e78f595b-4d67-447e-979b-65c4b66284e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 400, 'learning_rate': 1.0, 'algorithm': 'SAMME'}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AdaBoost_rs.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad120e00-c93b-41d3-84c4-8c09db56a458",
   "metadata": {},
   "source": [
    "**AdaBoost RANDOMIZED search Performance Evaluation**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "6af74730-083b-40a5-b908-c9157f48eae8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost RANDOMIZED Search Accuracy:\n",
      "\n",
      " 0.9883040935672515\n"
     ]
    }
   ],
   "source": [
    "ypred_AdaBoost_rs = AdaBoost_rs.predict(xtest)\n",
    "print('AdaBoost RANDOMIZED Search Accuracy:\\n\\n', accuracy_score(ytest,ypred_AdaBoost_rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "bb9409eb-acbf-442d-b281-71ca11f5ec51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost RANDOMIZED Search Classification Report:\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           B       0.99      0.99      0.99       108\n",
      "           M       0.98      0.98      0.98        63\n",
      "\n",
      "    accuracy                           0.99       171\n",
      "   macro avg       0.99      0.99      0.99       171\n",
      "weighted avg       0.99      0.99      0.99       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('AdaBoost RANDOMIZED Search Classification Report:\\n\\n', classification_report(ytest,ypred_AdaBoost_rs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf8d7b3a-1d49-4103-b45d-7d62ef712307",
   "metadata": {},
   "source": [
    "# XGBOOST Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4b8a879c-6a48-47b7-acd1-d0dc3040ce67",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "029d358f-69d8-4b2a-ad3e-b58838ef8310",
   "metadata": {},
   "outputs": [],
   "source": [
    "LE = LabelEncoder()\n",
    "y_encoded = LE.fit_transform(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "133d6ec7-a0c4-4aa4-b9f3-3bcf6a2b00e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "xtrain,xtest,ytrain,ytest = tts(x,y_encoded,test_size = 0.3, random_state = 42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "893958cf-f39e-4514-9bde-35892ccdc3f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-10 {color: black;}#sk-container-id-10 pre{padding: 0;}#sk-container-id-10 div.sk-toggleable {background-color: white;}#sk-container-id-10 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-10 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-10 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-10 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-10 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-10 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-10 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-10 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-10 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-10 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-10 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-10 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-10 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-10 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-10 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-10 div.sk-item {position: relative;z-index: 1;}#sk-container-id-10 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-10 div.sk-item::before, #sk-container-id-10 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-10 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-10 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-10 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-10 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-10 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-10 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-10 div.sk-label-container {text-align: center;}#sk-container-id-10 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-10 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-10\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-23\" type=\"checkbox\" checked><label for=\"sk-estimator-id-23\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgBoost = XGBClassifier(random_state = 42)\n",
    "xgBoost.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "1743afd5-ef7a-4d4a-a195-68d631245918",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred_xgboost = xgBoost.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "1d788209-060a-44bf-94e0-daa0dbad7a32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost Classifier Accuracy: \n",
      "\n",
      "Accuracy: 0.9824561403508771\n"
     ]
    }
   ],
   "source": [
    "print('XGBoost Classifier Accuracy:', '\\n\\nAccuracy:',accuracy_score(ytest, ypred_xgboost))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "4685f22c-9488-4d2b-ba14-236e5f00f199",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99       108\n",
      "           1       0.98      0.97      0.98        63\n",
      "\n",
      "    accuracy                           0.98       171\n",
      "   macro avg       0.98      0.98      0.98       171\n",
      "weighted avg       0.98      0.98      0.98       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification Report:\\n\\n', classification_report(ytest,ypred_xgboost))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86b51499-beb3-4199-a365-7f371bc51b95",
   "metadata": {},
   "source": [
    "# XGBOOST Hyperparameter Optimization and Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd9e38ee-205f-4c65-809f-8e43bcbffb66",
   "metadata": {},
   "source": [
    "**XGBoost with Grid Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d7f52013-1246-43bd-b8a4-bb7e84a68673",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'n_estimators': [100, 200, 300],\n",
    "    'learning_rate': [0.1, 0.01, 0.001],\n",
    "    'max_depth': [3, 5, 7],\n",
    "    'subsample': [0.8, 0.9],\n",
    "    'colsample_bytree': [0.7, 0.8],\n",
    "    'gamma': [0.0, 0.1, 0.2],\n",
    "    'min_child_weight': [1, 2],\n",
    "    'reg_alpha': [0.0, 0.1, 0.5],\n",
    "    'reg_lambda': [0.0, 0.1, 0.5],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "74fa425c-27be-4ca2-8a1e-a07a32738e59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-4 {color: black;}#sk-container-id-4 pre{padding: 0;}#sk-container-id-4 div.sk-toggleable {background-color: white;}#sk-container-id-4 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-4 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-4 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-4 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-4 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-4 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-4 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-4 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-4 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-4 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-4 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-4 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-4 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-4 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-4 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-4 div.sk-item {position: relative;z-index: 1;}#sk-container-id-4 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-4 div.sk-item::before, #sk-container-id-4 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-4 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-4 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-4 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-4 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-4 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-4 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-4 div.sk-label-container {text-align: center;}#sk-container-id-4 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-4 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-4\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None,...\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=42, ...),\n",
       "             param_grid={&#x27;colsample_bytree&#x27;: [0.7, 0.8],\n",
       "                         &#x27;gamma&#x27;: [0.0, 0.1, 0.2],\n",
       "                         &#x27;learning_rate&#x27;: [0.1, 0.01, 0.001],\n",
       "                         &#x27;max_depth&#x27;: [3, 5, 7], &#x27;min_child_weight&#x27;: [1, 2],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300],\n",
       "                         &#x27;reg_alpha&#x27;: [0.0, 0.1, 0.5],\n",
       "                         &#x27;reg_lambda&#x27;: [0.0, 0.1, 0.5],\n",
       "                         &#x27;subsample&#x27;: [0.8, 0.9]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None,...\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=42, ...),\n",
       "             param_grid={&#x27;colsample_bytree&#x27;: [0.7, 0.8],\n",
       "                         &#x27;gamma&#x27;: [0.0, 0.1, 0.2],\n",
       "                         &#x27;learning_rate&#x27;: [0.1, 0.01, 0.001],\n",
       "                         &#x27;max_depth&#x27;: [3, 5, 7], &#x27;min_child_weight&#x27;: [1, 2],\n",
       "                         &#x27;n_estimators&#x27;: [100, 200, 300],\n",
       "                         &#x27;reg_alpha&#x27;: [0.0, 0.1, 0.5],\n",
       "                         &#x27;reg_lambda&#x27;: [0.0, 0.1, 0.5],\n",
       "                         &#x27;subsample&#x27;: [0.8, 0.9]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=5,\n",
       "             estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                     callbacks=None, colsample_bylevel=None,\n",
       "                                     colsample_bynode=None,\n",
       "                                     colsample_bytree=None,\n",
       "                                     early_stopping_rounds=None,\n",
       "                                     enable_categorical=False, eval_metric=None,\n",
       "                                     feature_types=None, gamma=None,\n",
       "                                     gpu_id=None, grow_policy=None,\n",
       "                                     importance_type=None,\n",
       "                                     interaction_constraints=None,\n",
       "                                     learning_rate=None,...\n",
       "                                     missing=nan, monotone_constraints=None,\n",
       "                                     n_estimators=100, n_jobs=None,\n",
       "                                     num_parallel_tree=None, predictor=None,\n",
       "                                     random_state=42, ...),\n",
       "             param_grid={'colsample_bytree': [0.7, 0.8],\n",
       "                         'gamma': [0.0, 0.1, 0.2],\n",
       "                         'learning_rate': [0.1, 0.01, 0.001],\n",
       "                         'max_depth': [3, 5, 7], 'min_child_weight': [1, 2],\n",
       "                         'n_estimators': [100, 200, 300],\n",
       "                         'reg_alpha': [0.0, 0.1, 0.5],\n",
       "                         'reg_lambda': [0.0, 0.1, 0.5],\n",
       "                         'subsample': [0.8, 0.9]})"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_gs = GridSearchCV(XGBClassifier(random_state = 42),xgb_params, cv = 5)\n",
    "xgb_gs.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "56c1b2c9-5697-41f9-9382-63f7644816c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 0.7,\n",
       " 'gamma': 0.0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_depth': 7,\n",
       " 'min_child_weight': 1,\n",
       " 'n_estimators': 300,\n",
       " 'reg_alpha': 0.0,\n",
       " 'reg_lambda': 0.1,\n",
       " 'subsample': 0.8}"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "9be1b44b-e3d9-40b3-8094-2bf45718478a",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred_xgb_gs = xgb_gs.predict(xtest)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35606ea3-002c-46c8-9f89-21f5861dacfc",
   "metadata": {},
   "source": [
    "**XGBosst Evaluation and Accuracy for GRID Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "23771f3d-8f27-4151-b727-67ced6e02aeb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBOOST Accuracy: 0.9766081871345029\n"
     ]
    }
   ],
   "source": [
    "print('XGBOOST Accuracy:', accuracy_score(ytest,ypred_xgb_gs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "d7efbc23-102e-4e16-a5e9-4ffb182ec1b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification result of XGBOOST:\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98       108\n",
      "           1       0.98      0.95      0.97        63\n",
      "\n",
      "    accuracy                           0.98       171\n",
      "   macro avg       0.98      0.97      0.97       171\n",
      "weighted avg       0.98      0.98      0.98       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification result of XGBOOST:\\n\\n', classification_report(ytest,ypred_xgb_gs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0a13db2-d498-4970-a92c-66a00594fac9",
   "metadata": {},
   "source": [
    "**XGBOOST Randomized Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3c7538a6-b08f-4d7e-8bd5-4c62f0e32a0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomizedSearchCV(cv=5,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric=None, feature_types=None,\n",
       "                                           gamma=None, gpu_id=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning_rate...\n",
       "                                           monotone_constraints=None,\n",
       "                                           n_estimators=100, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           predictor=None, random_state=42, ...),\n",
       "                   param_distributions={&#x27;colsample_bytree&#x27;: [0.7, 0.8],\n",
       "                                        &#x27;gamma&#x27;: [0.0, 0.1, 0.2],\n",
       "                                        &#x27;learning_rate&#x27;: [0.1, 0.01, 0.001],\n",
       "                                        &#x27;max_depth&#x27;: [3, 5, 7],\n",
       "                                        &#x27;min_child_weight&#x27;: [1, 2],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300],\n",
       "                                        &#x27;reg_alpha&#x27;: [0.0, 0.1, 0.5],\n",
       "                                        &#x27;reg_lambda&#x27;: [0.0, 0.1, 0.5],\n",
       "                                        &#x27;subsample&#x27;: [0.8, 0.9]})</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-7\" type=\"checkbox\" ><label for=\"sk-estimator-id-7\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomizedSearchCV</label><div class=\"sk-toggleable__content\"><pre>RandomizedSearchCV(cv=5,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric=None, feature_types=None,\n",
       "                                           gamma=None, gpu_id=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning_rate...\n",
       "                                           monotone_constraints=None,\n",
       "                                           n_estimators=100, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           predictor=None, random_state=42, ...),\n",
       "                   param_distributions={&#x27;colsample_bytree&#x27;: [0.7, 0.8],\n",
       "                                        &#x27;gamma&#x27;: [0.0, 0.1, 0.2],\n",
       "                                        &#x27;learning_rate&#x27;: [0.1, 0.01, 0.001],\n",
       "                                        &#x27;max_depth&#x27;: [3, 5, 7],\n",
       "                                        &#x27;min_child_weight&#x27;: [1, 2],\n",
       "                                        &#x27;n_estimators&#x27;: [100, 200, 300],\n",
       "                                        &#x27;reg_alpha&#x27;: [0.0, 0.1, 0.5],\n",
       "                                        &#x27;reg_lambda&#x27;: [0.0, 0.1, 0.5],\n",
       "                                        &#x27;subsample&#x27;: [0.8, 0.9]})</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-8\" type=\"checkbox\" ><label for=\"sk-estimator-id-8\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-9\" type=\"checkbox\" ><label for=\"sk-estimator-id-9\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=None,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=None, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=None, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=100, n_jobs=None, num_parallel_tree=None,\n",
       "              predictor=None, random_state=42, ...)</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomizedSearchCV(cv=5,\n",
       "                   estimator=XGBClassifier(base_score=None, booster=None,\n",
       "                                           callbacks=None,\n",
       "                                           colsample_bylevel=None,\n",
       "                                           colsample_bynode=None,\n",
       "                                           colsample_bytree=None,\n",
       "                                           early_stopping_rounds=None,\n",
       "                                           enable_categorical=False,\n",
       "                                           eval_metric=None, feature_types=None,\n",
       "                                           gamma=None, gpu_id=None,\n",
       "                                           grow_policy=None,\n",
       "                                           importance_type=None,\n",
       "                                           interaction_constraints=None,\n",
       "                                           learning_rate...\n",
       "                                           monotone_constraints=None,\n",
       "                                           n_estimators=100, n_jobs=None,\n",
       "                                           num_parallel_tree=None,\n",
       "                                           predictor=None, random_state=42, ...),\n",
       "                   param_distributions={'colsample_bytree': [0.7, 0.8],\n",
       "                                        'gamma': [0.0, 0.1, 0.2],\n",
       "                                        'learning_rate': [0.1, 0.01, 0.001],\n",
       "                                        'max_depth': [3, 5, 7],\n",
       "                                        'min_child_weight': [1, 2],\n",
       "                                        'n_estimators': [100, 200, 300],\n",
       "                                        'reg_alpha': [0.0, 0.1, 0.5],\n",
       "                                        'reg_lambda': [0.0, 0.1, 0.5],\n",
       "                                        'subsample': [0.8, 0.9]})"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_rs = RandomizedSearchCV(estimator = XGBClassifier(random_state = 42), \n",
    "                            param_distributions = xgb_params, n_iter = 10, cv = 5)\n",
    "xgb_rs.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8c9bd1dd-d506-45a2-8fe7-77b01f7efc4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 0.8,\n",
       " 'reg_lambda': 0.0,\n",
       " 'reg_alpha': 0.1,\n",
       " 'n_estimators': 300,\n",
       " 'min_child_weight': 1,\n",
       " 'max_depth': 7,\n",
       " 'learning_rate': 0.1,\n",
       " 'gamma': 0.0,\n",
       " 'colsample_bytree': 0.7}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_rs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "77f41012-fd33-4892-843b-eee15b3ae93f",
   "metadata": {},
   "outputs": [],
   "source": [
    "ypred_xgb_rs = xgb_rs.predict(xtest)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "689b7a67-2788-4999-b67c-657d5bceca55",
   "metadata": {},
   "source": [
    "**XGBosst Evaluation and Accuracy for RANDOMIZED Search**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3a779fc9-5e29-4d22-a1c1-6d6948d51e1c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBOOST Randomized Search Accuracy: 0.9707602339181286\n"
     ]
    }
   ],
   "source": [
    "print('XGBOOST Randomized Search Accuracy:', accuracy_score(ytest,ypred_xgb_rs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0c4c2e96-2960-412f-b4e0-c76a92009134",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification result of XGBOOST (Randomized Search):\n",
      "\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.98      0.98       108\n",
      "           1       0.97      0.95      0.96        63\n",
      "\n",
      "    accuracy                           0.97       171\n",
      "   macro avg       0.97      0.97      0.97       171\n",
      "weighted avg       0.97      0.97      0.97       171\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Classification result of XGBOOST (Randomized Search):\\n\\n', classification_report(ytest,ypred_xgb_rs))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}