14_balanced_RF_model_precision.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Balanced random forest model with precision score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we use precision as the target score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Some variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapath = '/data/ml_course/05_Capstone_project/rf_data/'\n",
    "model_name = 'balanced_RF_precision.pickle'\n",
    "test_size = 0.1 # 10% of points used for testing\n",
    "val_size = 0.1  # 10% of points (after removing test points) used for validation\n",
    "n_splits = 5  # number of splits in folding\n",
    "n_estimators = np.r_[np.arange(1, 22, 2), np.arange(20, 60, 10)]  # number of trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# suppress anoying iypthon warnings. Not ideal since we suppress also potentially relevant warnings\n",
    "#import warnings\n",
    "#warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "202009\r"
     ]
    }
   ],
   "source": [
    "years = ['2018', '2019', '2020']\n",
    "months = ['04', '05', '06', '07', '08', '09']\n",
    "\n",
    "for year in years:\n",
    "    for month in months:\n",
    "        print(year+month, end=\"\\r\", flush=True)\n",
    "        fname = datapath+year+month+'_data.npz'\n",
    "        if not os.path.isfile(fname):\n",
    "            continue\n",
    "        \n",
    "        X = None\n",
    "        with np.load(fname, allow_pickle=False) as npz_file:\n",
    "            # Load the arrays\n",
    "            if X is None:\n",
    "                X = npz_file['features']\n",
    "                y = npz_file['targets']\n",
    "            else:\n",
    "                X = np.concatenate((X, npz_file['features']), axis=0)\n",
    "                y = np.concatenate((y, npz_file['targets']), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features matrix shape: (7435724, 6)\n",
      "target matrix shape: (7435724,)\n"
     ]
    }
   ],
   "source": [
    "print('features matrix shape:', X.shape)\n",
    "print('target matrix shape:', y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove NaNs\n",
    "# NaNs can appear in texture fields.\n",
    "# They could safely be considered 0s but we remove them\n",
    "ind_row, ind_col = np.where(np.isnan(X))\n",
    "X = np.delete(X, ind_row, axis=0)\n",
    "y = np.delete(y, ind_row, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features matrix shape: (7432290, 6)\n",
      "target matrix shape: (7432290,)\n"
     ]
    }
   ],
   "source": [
    "print('features matrix shape:', X.shape)\n",
    "print('target matrix shape:', y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1], dtype=int8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group together 0 (No POH was computed) and 1 (POH below 90%)\n",
    "np.unique(y)\n",
    "y[y == 1] = 0\n",
    "y[y == 2] = 1\n",
    "np.unique(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "npixels: 7432290\n",
      "npixels with hail: 61171\n",
      "% of pixels with hail: 0.8230437725115678\n"
     ]
    }
   ],
   "source": [
    "npixels = y.size\n",
    "npixels_hail = y[y==1].size\n",
    "\n",
    "print('npixels:', npixels)\n",
    "print('npixels with hail:', npixels_hail)\n",
    "print('% of pixels with hail:', npixels_hail/npixels*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split data into test and training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split data\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=test_size, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "features matrix for training shape: (6689061, 6)\n",
      "target matrix for training shape: (6689061,)\n"
     ]
    }
   ],
   "source": [
    "print('features matrix for training shape:', X_tr.shape)\n",
    "print('target matrix for training shape:', y_tr.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss = ShuffleSplit(n_splits=n_splits, test_size=val_size, random_state=0)\n",
    "ss.get_n_splits()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 15 candidates, totalling 75 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed: 17.2min\n",
      "[Parallel(n_jobs=-1)]: Done  75 out of  75 | elapsed: 52.2min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=ShuffleSplit(n_splits=5, random_state=0, test_size=0.1, train_size=None),\n",
       "             estimator=Pipeline(steps=[('scaler', None),\n",
       "                                       ('RF',\n",
       "                                        BalancedRandomForestClassifier(random_state=0))]),\n",
       "             n_jobs=-1,\n",
       "             param_grid={'RF__n_estimators': array([ 1,  3,  5,  7,  9, 11, 13, 15, 17, 19, 21, 20, 30, 40, 50])},\n",
       "             return_train_score=True, scoring='precision', verbose=True)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create pipeline\n",
    "# Balanced random forest\n",
    "pipe_RF = Pipeline([\n",
    "    ('scaler', None),\n",
    "    ('RF', BalancedRandomForestClassifier(random_state=0, max_depth=None))\n",
    "])\n",
    "\n",
    "# Create cross-validation object\n",
    "grid_RF = {'RF__n_estimators': n_estimators}\n",
    "\n",
    "grid_RF_cv = GridSearchCV(pipe_RF, grid_RF, scoring='precision', cv=ss, return_train_score=True, verbose=True, n_jobs=-1)\n",
    "\n",
    "# Fit estimator\n",
    "grid_RF_cv.fit(X_tr, y_tr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Collect results in a DataFrame\n",
    "RF_results = pd.DataFrame({\n",
    "    'trees': grid_RF_cv.cv_results_['param_RF__n_estimators'],\n",
    "    'mean_tr': grid_RF_cv.cv_results_['mean_train_score'],\n",
    "    'mean_te': grid_RF_cv.cv_results_['mean_test_score'],\n",
    "    'std_te': grid_RF_cv.cv_results_['std_test_score']\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['mean_fit_time', 'std_fit_time', 'mean_score_time', 'std_score_time', 'param_RF__n_estimators', 'params', 'split0_test_score', 'split1_test_score', 'split2_test_score', 'split3_test_score', 'split4_test_score', 'mean_test_score', 'std_test_score', 'rank_test_score', 'split0_train_score', 'split1_train_score', 'split2_train_score', 'split3_train_score', 'split4_train_score', 'mean_train_score', 'std_train_score'])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_RF_cv.cv_results_.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEGCAYAAABsLkJ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAApLUlEQVR4nO3df3RU9Z3/8ec7k18QAuFH+BkwVAEFfyBGaotbiz9BXbG7213tWl27Z6mttrZnu65+u62263br1uNaW4+U49LqUevW06rU0iJV6a5uaYWCCgIaMYSQQCCBEBLy+/39Y25gGAZyCQlD5r4e5+TM3Hs/d+bzoXVecz/3M5+PuTsiIhI9WemugIiIpIcCQEQkohQAIiIRpQAQEYkoBYCISERlp7sCx2PUqFFeWlqa7mqIiAwoa9as2e3uxcn7B1QAlJaWsnr16nRXQ0RkQDGzran2qwtIRCSiFAAiIhGlABARiagBdQ8glfb2dqqqqmhpaUl3VQaU/Px8SkpKyMnJSXdVRCRNBnwAVFVVUVhYSGlpKWaW7uoMCO5OXV0dVVVVTJ48Od3VEZE0GfBdQC0tLYwcOVIf/sfBzBg5cqSumkQibsAHAKAP/17Qv5mIhAoAM5tnZpvNrNzM7k5x3MzskeD422Y2K+FYhZm9Y2brzOyIQfxm9jUzczMbdWJNkaiprGvmR7/7gF/8qYrff1DH1romWjs6010tkQGjx3sAZhYDHgWuAKqAN81sqbu/m1BsPjAl+Pso8Fjw2G2uu+9O8doTg9et7HULBICHH36YhQsXMnjw4HRX5aT5xdoqHv7t+0fsHzUkl3HDBjFuWD7ji+KP44oGMT54HFOYR3YsIy5+RU5ImJvAs4Fyd98CYGbPAguAxABYADzp8dVlVplZkZmNc/eaHl77P4G7gBePv+qS6OGHH+amm26KVACUjiwA4EefvYAhedlU7z1ATUMLNQ0HqN7bQkVdE7//oI7G1o7DzssyGF2Yz7iifMYPOzIgxg/LZ9SQPLKy1E0mmS1MAEwAtiVsV3H4t/ujlZkA1AAOvGxmDvzI3RcDmNl1wHZ3f+tY/dFmthBYCDBp0qQQ1T35KioqmDdvHhdffDGrVq3ivPPO49Zbb+Xee++ltraWp59+mhkzZvClL32Jd955h46ODu677z4WLFhARUUFn/3sZ2lqagLghz/8IR//+MdZuXIl9913H6NGjWL9+vVccMEFPPXUUyn77h955BGqq6uZO3cuo0aN4rXXXuPll1/m3nvvpbW1ldNPP50f//jHDBky5GT/0/SraWMLAWhp7+SqGWOPWq6xpZ2ahpZDAbH3ANVBUGys2ccrm3bS0t512Dk5MWPM0CAgivIZN2wQ44PH7iuL4YNzdC9FBrQwAZDq/+HJ60geq8wcd682s9HACjPbBKwGvg5c2dObB4GxGKCsrOyY61d+65cbeLd6X08veVymjx/KvX8+o8dy5eXlPPfccyxevJgLL7yQZ555htdff52lS5fyne98h+nTp3PppZeyZMkS9u7dy+zZs7n88ssZPXo0K1asID8/n/fff58bb7zx4HxHa9euZcOGDYwfP545c+bwxhtvcPHFFx/x3l/+8pd56KGHeO211xg1ahS7d+/m/vvv57e//S0FBQU88MADPPTQQ3zzm9/s03+bdDu9eAjZWcbmHY3HLFeYn0Nhfg5TxxSmPO7u7G1up7rhADV7gyuIhKD4U+UedjTU0N55+P/98nOyDgbCYQFRdCg4hubrdxZy6goTAFXAxITtEqA6bBl3736sNbPniXcp7QEmA93f/kuAP5nZbHff0Yt2pN3kyZM555xzAJgxYwaXXXYZZsY555xDRUUFVVVVLF26lAcffBCID1+trKxk/Pjx3HHHHaxbt45YLMZ777138DVnz55NSUkJADNnzqSioiJlACRbtWoV7777LnPmzAGgra2Nj33sY33d5LTLzc7i9OIhPQZAT8yM4QW5DC/IZcb4YSnLdHU5u5taDwVEUlD83we72bmvha6kryhD8rIP72JKCojxwwYxKDd2QvUX6a0wAfAmMMXMJgPbgRuAzySVWQrcEdwf+CjQ4O41ZlYAZLl7Y/D8SuDb7v4OMLr7ZDOrAMpS3Sg+HmG+qfeXvLy8g8+zsrIObmdlZdHR0UEsFuPnP/8506ZNO+y8++67jzFjxvDWW2/R1dVFfn5+yteMxWJ0dBzel3007s4VV1zBT3/60xNp0oAwbWwha7bu6ff3ycoyRhfmM7own/MmFqUs09HZRW1j6+EBETzWNLTwbvU+du9vPeK8osE58SuIYflHdDeNHzaIMcPyyMtWSEjf6zEA3L3DzO4AlgMxYIm7bzCz24Lji4BlwNVAOdAM3BqcPgZ4PviWnw084+6/6fNWDABXXXUVP/jBD/jBD36AmbF27VrOP/98GhoaKCkpISsriyeeeILOzt4NYywsLKSxsZFRo0Zx0UUXcfvtt1NeXs4ZZ5xBc3MzVVVVTJ06tY9blX7Txhay9K1q9rW0p727JTuWxfiiQYwvGsQFp6Uu09rRyc6G1nh3U2JA7G2huqGF1Vv30HCg/YjzRg3JC0IhHgxjhuYzdFA2Q/KyKczPZkheTvCYffBRI52kJ6GmgnD3ZcQ/5BP3LUp47sDtKc7bApwX4vVLw9RjIPvGN77BV77yFc4991zcndLSUl566SW++MUv8pd/+Zc899xzzJ07l4KCgl69/sKFC5k/fz7jxo3jtdde4yc/+Qk33ngjra3xb5z3339/RgbAmcGN4Pd2NFJWOiLNtelZXnaMSSMHM2nk0UdrNbd1JAXDoccPdjXx+vu7aWrr+YvCoJwYQ/KzKczLjj8GwdAdFge3g8eh+TkHn3dvF+TFFCQZzOKf3QNDWVmZJy8Is3HjRs4666w01Whgy4R/u6o9zVz8wGvcf/3Z3HTRUb52Zxh350B7J40tHTS2dLC/tYP9LR00trTTGDzf3xrfjj92HHpMOpZ8zyKVg0GSECbxK42cw644CoMASQ6cwjwFSbqZ2Rp3L0veP+Ang5Nom1A0iMK87BO+ETyQmBmDc7MZnJvNmKG9fx13p7mtMykk2uNhcjBUOtjfeihIusvtbmxmf2sH+4IgCfM9cnBu7OAVR2FCgBy6Aul+nnPMwInp9xl9RgEwwHzqU5/iww8/PGzfAw88wFVXXZWmGqWXmTFtbCGbdvTt8N8oMDMK8rIpyOvLIGlPuirpSLgqaU/a7qC2seVQ4BxHkCRedSR2ZyVelcT/Um8Pzo3pNxwoAAac559/Pt1VOOV03wh2d/1HnQaHB0l+zyccRVfXoa6t7rBI7LpqTLxC6T4W7NvR0HIoeFp7Hi2XZRwWIAe7sroDJVXXVn68O6sw/1AX10AfnZURAaD/8I/fQLr305Mzxxby9B86qGloYXzRoHRXR3opK+tQkEDvg6Szy2lqSwiO4N5I4nZil1b39u79bVTUNR/c19rR1eN75cayDr/fEXRhDc1PvA+Sk3AFcviIrXTfaB/wAZCfn09dXZ3WBDgO3QvCJP7mYCCbNjbef7F5R6MCQIhlGUPzc054WHBbR9fBrqx9h91Qb08Ij4SrlWB7+94DbEoo3xniTvth90fyc1JegXzq/AlMHtW7UYJHM+ADoKSkhKqqKnbt2pXuqgwo3UtCZoJpwRQPm3Y0MvfM0T2UFgknNzuLEdm5jCjI7fVruDst7V00JoXE/tZ29iVtNx7s5upgf0s7O/e1HHaD/sLS4QqAZDk5OVrWMOKGDc5h/LB8NutGsJxizIxBuTEG5cYYnXoqqlC6uvyICdj6woAPABEgGAkUnaGgEi39NTW5fpkhGWHa2KF8sGs/7Z0937gTkTgFgGSEM8cW0t7pbNnVlO6qiAwYCgDJCN2Lw+gHYSLhKQAkI4RdHEZEDlEASEboq8VhRKJEASAZQyOBRI6PAkAyxrSxhWzfe4B9LUcuqCIiRwoVAGY2z8w2m1m5md2d4riZ2SPB8bfNbFbCsQoze8fM1pnZ6oT93zOzTUH5582sqE9aJJGVuDiMiPSsxwAwsxjwKDAfmA7caGbTk4rNB6YEfwuBx5KOz3X3mUkLEqwAznb3c4H3gHt61wSRuEMjgRQAImGEuQKYDZS7+xZ3bwOeBRYklVkAPOlxq4AiMxt3rBd195fdvXve1lVAZkxMI2kTxcVhRE5EmACYAGxL2K4K9oUt48DLZrbGzBYe5T0+B/w61QEzW2hmq81stSZ8k2PR4jAixydMAKSahCJ5XqJjlZnj7rOIdxPdbmafOOxEs68DHcDTqd7c3Re7e5m7lxUXF4eorkRZ90igTFrvQKS/hAmAKmBiwnYJUB22jLt3P9YCzxPvUgLAzG4BrgX+1vVfrPSBM8cW0tgSXxxGRI4tTAC8CUwxs8lmlgvcACxNKrMUuDkYDXQR0ODuNWZWYGaFAGZWAFwJrA+25wH/DFzn7s191B6JuMTFYUTk2HqcDtrdO8zsDmA5EAOWuPsGM7stOL4IWAZcDZQDzcCtweljgOeDlbqygWfc/TfBsR8CecCK4Pgqd7+trxom0ZQ4EkiLw4gcW6j1ANx9GfEP+cR9ixKeO3B7ivO2AOcd5TXPOK6aioQwbJAWhxEJS78EloyjKSFEwlEASMZwd3bvb2V4Qa4WhxEJQUtCyoDS2NLOtvoDbNvTzLb6Zqr2HGBbfXOwfYAD7Z0AZBnsb+lg+Aks6C2S6RQA0u9aOzr5oLaJzTv3sWlHI1t2NR327Tx5AHDyeGB3Z29zO9v2NLO3+fCJ3obkZVMyfBCnjSzg4jOKmThiEBOHD2ba2EJ9+Iv0QAEgfcbdqdpzgM07Gtm8s5GNNfvYvKORD3c30dEV/1jPjWVROmowg3Jih59sh/+WMPmXhcMLcjm3ZBgTRwxm4vDBBz/oiwbnYNY/C2aLZDoFgPRafVMbKzfXsnrrHjbV7OO9nfvZ39px8HjJ8EGcObaQK2eMYdrYoZw1tpDSUQXkxHTrSeRUoACQ0NydjTWNvLppJ69uqmXttr24x4deThtbyF/MmsCZY4cybWwhU8cMoTA/J91VFpFjUADIMR1o6+SN8t28urmW1zbVHpxi4bySYdx52RQuO3MMM8YPJStL3TAiA40CQI5QtaeZ1zbV8sqmWn7/QR2tHV0U5Mb4synFfPWK0XxyWjGjC/PTXU0ROUEKAMHd2bSjkV+v38HLG3Yc/BHVaSMH85mPTuKyM8dw4eTh5GXHenglERlIFAAR5e68VdXAr9fXsHz9DirqmjGDC0tH8C/XnMXcM0fzkVEFGmEjksEUABHS2eWsrqjn1+t3sHzDDmoaWsjOMj52+kgWfuJ0rpg+huLCvHRXU0ROEgVAhmvv7OL3H9Tx6/U7WPHuDnbvbyM3O4tLphbztSuncflZYxg2WKN1RKJIAZChmts6WPS7LfzkjQ/Z19JBQW6MuWeOZt7ZY5k7bTQFefqfXiTq9CmQYdydX75dw78v20hNQwvzzx7LX8wq4c+mjCI/+de3IhJpCoAMsn57A9/65QberNjD2ROG8siN53Nh6Yh0V0tETlGhAiBYvvH7xFcEe9zdv5t03ILjVxNfEezv3P1PwbEKoBHoBDrcvSzYPwL4b6AUqAD+2t33nHCLImj3/lYeXL6Z/169jRGDc/nuX5zDp8smEtOPs0TkGHoMADOLAY8CVxBf/P1NM1vq7u8mFJsPTAn+Pgo8Fjx2m+vuu5Ne+m7gFXf/rpndHWz/c69bEkHtnV088X8VfP+V9znQ1snfz5nMly6bwrBBuqkrIj0LcwUwGygPlnfEzJ4FFgCJAbAAeDJYGnKVmRWZ2Th3rznG6y4APhk8fwJYiQIgtJWba/nXl97lg11NXDK1mG9cO50zRg9Jd7VEZAAJEwATgG0J21Uc/u3+aGUmADXEp3d/2cwc+JG7Lw7KjOkOCHevMbOUK3ib2UJgIcCkSZNCVDezfbi7iftfepdXNtUyeVQBS/6ujLnTRusHWyJy3MIEQKpPluQ1O45VZo67Vwcf8CvMbJO7/0/YCgaBsRigrKws+X0jo6m1g0defZ8lr39IXnaMe+afya1zJpObramVRaR3wgRAFTAxYbsEqA5bxt27H2vN7HniXUr/A+zs7iYys3FAbe+akNncneUbdvLtX26guqGFv7qghLvmTdNkbCJywsJ8fXwTmGJmk80sF7gBWJpUZilws8VdBDQEH+wFZlYIYGYFwJXA+oRzbgme3wK8eIJtyTiVdc187idvcttTaxg6KIeff+FjPPjp8/ThLyJ9oscrAHfvMLM7gOXEh4EucfcNZnZbcHwRsIz4ENBy4sNAbw1OHwM8H/RPZwPPuPtvgmPfBX5mZn8PVAKf7rNWDXCtHZ0s/t0WfvhaOdlZxr9ccxZ/9/FSsrWSloj0IfPkFblPYWVlZb569ep0V6Nfvf7+br7x4no+3N3ENeeO4xvXTGfsMH3jF5HeM7M13b/BSqRfAp8idu5r4V9fepeX3q6hdORgnvzcbD4xtTjd1RKRDKYASLOOzi6e/P1WHlrxHm2dXXz18ql8/pKPaN4eEel3CoA0WrN1D//ywno21uzjkqnFfHvBDE4bWZDuaolIRCgA0qCzy3n4t+/xg1fLGTcsn0U3zeKqGWP1Yy4ROakUACfZ3uY27nx2Hb97bxd/XVbCvX8+Q3Pzi0ha6JPnJHq3eh+ff2o1Oxpa+M6nzuEzH9XUFiKSPgqAk+SFtdu5+xdvUzQol//+/MeYNWl4uqskIhGnAOhn7Z1dfGfZRn78RgWzJ4/g0c/M0sLrInJKUAD0o9rGFu54ei1/rKjnc3Mmc8/VZ5KjX/OKyClCAdBP1mzdwxefXkPDgXYe/puZXH/+hHRXSUTkMAqAPubuPPPHSu5buoGxw/L5xRfmMH380HRXS0TkCAqAPtTS3sk3X1zPz1ZXccnUYr5/w0yKBuemu1oiIikpAPpI9d4D3PbUGt6uauBLl57BVy6fqkXZReSUpgDoI1/+6Vq27Gpi8Wcv4MoZY9NdHRGRHmlISh+o2N3E6q17uOPSM/ThLyIDhgKgD7ywbjtmcN1549NdFRGR0EIFgJnNM7PNZlZuZnenOG5m9khw/G0zm5V0PGZma83spYR9M81slZmtM7PVZjb7xJtz8rk7L66r5qLJIxlfNCjd1RERCa3HADCzGPAoMB+YDtxoZtOTis0HpgR/C4HHko7fCWxM2vcfwLfcfSbwzWB7wHmrqoEPdzfxKY3zF5EBJswVwGyg3N23uHsb8CywIKnMAuBJj1sFFJnZOAAzKwGuAR5POseB7gHyw4DqXrYhrV5Yu53c7CzmnaO+fxEZWMKMApoAbEvYrgI+GqLMBKAGeBi4CyhMOucrwHIze5B4EH081Zub2ULiVxVMmnRqzZ7Z3tnFL9+q5vKzRjM0Pyfd1REROS5hrgBSDWZPXkk+ZRkzuxaodfc1KY5/Afiqu08Evgr8V6o3d/fF7l7m7mXFxafWGrmvl++mrqmN62eq+0dEBp4wAVAFTEzYLuHI7pqjlZkDXGdmFcS7ji41s6eCMrcAvwieP0e8q2lAeWHtdooG5/DJaaPTXRURkeMWJgDeBKaY2WQzywVuAJYmlVkK3ByMBroIaHD3Gne/x91L3L00OO9Vd78pOKcauCR4finw/ok25mTa39rB8g07uOacceRmazStiAw8Pd4DcPcOM7sDWA7EgCXuvsHMbguOLwKWAVcD5UAzcGuI9/4H4Ptmlg20EPTzDxQvb9hBS3uXZvkUkQEr1FQQ7r6M+Id84r5FCc8duL2H11gJrEzYfh24IHxVTy0vrKumZPggLtDKXiIyQKnvohdqG1t4/f1dXD9zAlma8E1EBigFQC/88q0auhyuP19TP4jIwKUA6IUX1m7n7AlDOWN08k8bREQGDgXAcSqv3c872xs09l9EBjwFwHF6cd12sjTzp4hkAAXAcXB3nl+7nTlnjGL00Px0V0dE5IQoAI7Dmq17qNpzQDN/ikhGUAAch+fXbic/J0urfolIRlAAhNTW0cWv3qnhyuljGZKnpZRFZOBTAIT0u/d2sbe5Xd0/IpIxFAAhvbB2OyMLcrl4yqh0V0VEpE8oAELY19LOio07ufbcceTE9E8mIplBn2Yh/Gb9Dto6NPOniGQWBUAIL6zdTunIwcycWJTuqoiI9BkFQA9qGg7w+y11XH/+BMw086eIZA4FQA+WrqvGHc39IyIZJ1QAmNk8M9tsZuVmdneK42ZmjwTH3zazWUnHY2a21sxeStr/peB1N5jZf5xYU/rHC+uqmTmxiNJRBemuiohIn+oxAMwsBjwKzAemAzea2fSkYvOBKcHfQuCxpON3AhuTXncusAA4191nAA/2pgH9adOOfWys2aex/yKSkcJcAcwGyt19i7u3Ac8S/+BOtAB40uNWAUVmNg7AzEqAa4DHk875AvBdd28FcPfaE2hHv3hhbTWxLOPac8eluyoiIn0uTABMALYlbFcF+8KWeRi4C+hKOmcq8Gdm9gcz+52ZXZjqzc1soZmtNrPVu3btClHdvtHV5Sxdt51LphYzckjeSXtfEZGTJUwApBr64mHKmNm1QK27r0lxPBsYDlwE/BPwM0sxzMbdF7t7mbuXFRcXh6hu3/hjRT3VDS0smKl5/0UkM4UJgCpgYsJ2CVAdsswc4DozqyDedXSpmT2VcM4vgm6jPxK/Qjhl5ll4Ye12CnJjXDldM3+KSGYKEwBvAlPMbLKZ5QI3AEuTyiwFbg5GA10ENLh7jbvf4+4l7l4anPequ98UnPMCcCmAmU0FcoHdJ9yiPtDS3smv3qnhqrPHMig3lu7qiIj0ix7nNXb3DjO7A1gOxIAl7r7BzG4Lji8ClgFXA+VAM3BriPdeAiwxs/VAG3CLuyd3LaXFxpp9NLZ0cOX0MemuiohIvwk1sb27LyP+IZ+4b1HCcwdu7+E1VgIrE7bbgJuOVj6dKuubAfhI8ZA010REpP/ol8ApVNbFA2Di8MFpromISP9RAKSwtb6Z0YV56v8XkYymAEihsr6Z00bq27+IZDYFQAqVdc1MHKEAEJHMpgBI0tLeyY59LZw2QpO/iUhmUwAkqdoTvwGsLiARyXQKgCRbu0cAqQtIRDKcAiBJdwDoCkBEMp0CIEllfTMFuTFGFuSmuyoiIv1KAZCksj4+Akjr/4pIplMAJNla16TuHxGJBAVAgq4uZ9ueA5w2UkNARSTzKQAS7Gxsoa2jSyOARCQSFAAJDo4AUgCISAQoABJ0TwOtewAiEgUKgASVdc3EsozxRYPSXRURkX4XKgDMbJ6ZbTazcjO7O8VxM7NHguNvm9mspOMxM1trZi+lOPdrZuZmlvb1gLfWNzO+KJ+cmHJRRDJfj590ZhYDHgXmA9OBG81selKx+cCU4G8h8FjS8TuBjSleeyJwBVB53DXvB5X1zZoETkQiI8xX3dlAubtvCZZxfBZYkFRmAfCkx60CisxsHICZlQDXAI+neO3/BO4CTom1gCvrmjQCSEQiI0wATAC2JWxXBfvClnmY+Id8V+IJZnYdsN3d3zrWm5vZQjNbbWard+3aFaK6vbOvpZ09ze26ASwikREmAFLNiZD8jT1lGTO7Fqh19zWHFTYbDHwd+GZPb+7ui929zN3LiouLQ1S3dyo1BFREIiZMAFQBExO2S4DqkGXmANeZWQXxrqNLzewp4HRgMvBWcKwE+JOZje1FG/pE9xBQdQGJSFSECYA3gSlmNtnMcoEbgKVJZZYCNwejgS4CGty9xt3vcfcSdy8NznvV3W9y93fcfbS7lwbHqoBZ7r6jz1p2nDQNtIhETXZPBdy9w8zuAJYDMWCJu28ws9uC44uAZcDVQDnQDNzaf1XuH5X1zYwoyKUwPyfdVREROSl6DAAAd19G/EM+cd+ihOcO3N7Da6wEVh7lWGmYevSnynqNABKRaNEvngJb65p1A1hEIkUBALR3dlG994D6/0UkUhQAwPY9B+hyjQASkWhRABCfAwj0GwARiRYFAInTQGseIBGJDgUA8TmAcrOzGF2Yl+6qiIicNAoA4iOAJo0YTFZWqhktREQykwKA7mmg1f8vItES+QBwdyrrmzUCSEQiJ/IBsHt/G81tnfoNgIhETuQDQAvBi0hUKQDqmwCYpC4gEYmYyAfA1rpmzKBkuAJARKIl8gFQWd/M2KH55OfE0l0VEZGTSgFQpxFAIhJNkQ+ArfoNgIhEVKgAMLN5ZrbZzMrN7O4Ux83MHgmOv21ms5KOx8xsrZm9lLDve2a2KSj/vJkVnXBrjtOBtk52NbZqBJCIRFKPAWBmMeBRYD4wHbjRzKYnFZsPTAn+FgKPJR2/E9iYtG8FcLa7nwu8B9xz3LU/QVoIXkSiLMwVwGyg3N23uHsb8CywIKnMAuBJj1sFFJnZOAAzKwGuAR5PPMHdX3b3jmBzFVByAu3ola118SGgmgVURKIoTABMALYlbFcF+8KWeRi4C+g6xnt8Dvh1iLr0qUqtAyAiERYmAFJNkelhypjZtUCtu6856oubfR3oAJ4+yvGFZrbazFbv2rUrRHXDq6xvpjAvm6LBOX36uiIiA0GYAKgCJiZslwDVIcvMAa4zswriXUeXmtlT3YXM7BbgWuBv3T05VABw98XuXubuZcXFxSGqG97WumYmjRyMmaaBFpHoCRMAbwJTzGyymeUCNwBLk8osBW4ORgNdBDS4e4273+PuJe5eGpz3qrvfBPGRRcA/A9e5e3NfNeh4bKtv1gggEYms7J4KuHuHmd0BLAdiwBJ332BmtwXHFwHLgKuBcqAZuDXEe/8QyANWBN/AV7n7bb1qRS90djnb9jRzxYwxJ+stRUROKT0GAIC7LyP+IZ+4b1HCcwdu7+E1VgIrE7bPOI569rmahgO0dzqnjdAIIBGJpsj+EljTQItI1EU3AOriAaBpoEUkqiIbAFvrm8nOMsYNy093VURE0iKyAVBZ30zJ8EFkxyL7TyAiERfZTz9NAy0iURfZANha16QbwCISaZEMgIbmdva1dGgIqIhEWiQDYGuwELy6gEQkyqIZAHX6DYCISCQDoPtHYPoNgIhEWTQDoK6ZUUNyKcgLNROGiEhGimQAbK1v0rd/EYm8SAbAtvoDWgZSRCIvcgHQ2tFJdcMBjQASkciLXABU7TmAu9YBFhGJXABoGmgRkbhQAWBm88xss5mVm9ndKY6bmT0SHH/bzGYlHY+Z2Vozeylh3wgzW2Fm7wePw0+8OT3TNNAiInE9BoCZxYBHgfnAdOBGM5ueVGw+MCX4Wwg8lnT8TmBj0r67gVfcfQrwSrDd7yrrmxmUE6O4MO9kvJ2IyCkrzBXAbKDc3be4exvwLLAgqcwC4EmPWwUUmdk4ADMrAa4BHk9xzhPB8yeA63vXhOOzta6ZSSMGE6xDLCISWWECYAKwLWG7KtgXtszDwF1AV9I5Y9y9BiB4HB2uyiemsr5JI4BERAgXAKm+KnuYMmZ2LVDr7muOu2bdL2y20MxWm9nqXbt29fZl4hVyp7K+WTeARUQIFwBVwMSE7RKgOmSZOcB1ZlZBvOvoUjN7KiizM6GbaBxQm+rN3X2xu5e5e1lxcXGI6h7drsZWWtq7FAAiIoQLgDeBKWY22cxygRuApUlllgI3B6OBLgIa3L3G3e9x9xJ3Lw3Oe9Xdb0o455bg+S3AiyfamJ5sDYaAqgtIRAR6nA3N3TvM7A5gORADlrj7BjO7LTi+CFgGXA2UA83ArSHe+7vAz8zs74FK4NO9a0J43UNA9SMwEZEQAQDg7suIf8gn7luU8NyB23t4jZXAyoTtOuCy8FU9cVvrmzGDkuEKABGRSP0SuLKuifHDBpGbHalmi4ikFKlPwsr6Zv0CWEQkoAAQEYmoyATA/tYOdu9vY5KGgIqIABEKgG2aBVRE5DCRCYCtmgVUROQwkQmAyvomAE4boaUgRUQgUgHQzLBBOQwbnJPuqoiInBIiEwDd00CLiEhcZAKgsr5ZI4BERBJEIgA6OrvYvueA5gASEUkQiQCoaWiho8vVBSQikiASAXBwCKi6gEREDopEAFQe/BGYhoCKiHSLRABsrW8iJ2aMHZqf7qqIiJwyIhEAk0cW8KnzJxDLSrV0sYhINIVaEGagu2H2JG6YPSnd1RAROaWEugIws3lmttnMys3s7hTHzcweCY6/bWazgv35ZvZHM3vLzDaY2bcSzplpZqvMbJ2ZrTaz2X3XLBER6UmPAWBmMeBRYD4wHbjRzKYnFZsPTAn+FgKPBftbgUvd/TxgJjAvWDQe4D+Ab7n7TOCbwbaIiJwkYa4AZgPl7r7F3duAZ4EFSWUWAE963CqgyMzGBdv7gzI5wZ8H2w4MDZ4PA6pPpCEiInJ8wtwDmABsS9iuAj4aoswEoCa4glgDnAE86u5/CMp8BVhuZg8SD6KPp3pzM1tI/KqCSZPUjy8i0lfCXAGkGjrjYcu4e2fQzVMCzDazs4PjXwC+6u4Tga8C/5Xqzd19sbuXuXtZcXFxiOqKiEgYYQKgCpiYsF3Ckd01PZZx973ASmBesOsW4BfB8+eIdzWJiMhJEiYA3gSmmNlkM8sFbgCWJpVZCtwcjAa6CGhw9xozKzazIgAzGwRcDmwKzqkGLgmeXwq8f2JNERGR49HjPQB37zCzO4DlQAxY4u4bzOy24PgiYBlwNVAONAO3BqePA54I7gNkAT9z95eCY/8AfN/MsoEWgn5+ERE5Ocw9uTv/1GVmu4CtPRQbBew+CdU51ajd0aJ2R8+JtP00dz/iJuqACoAwzGy1u5elux4nm9odLWp39PRH2yMxF5CIiBxJASAiElGZGACL012BNFG7o0Xtjp4+b3vG3QMQEZFwMvEKQEREQlAAiIhEVMYEQE9rFmQSM1tiZrVmtj5h3wgzW2Fm7wePw9NZx/5gZhPN7DUz2xisL3FnsD+j2360dTUyvd0Qn47ezNaa2UvBdsa3GcDMKszsne71UoJ9fd72jAiAkGsWZJKfcGhOpW53A6+4+xTglWA703QA/+juZwEXAbcH/ztnetuPtq5Gprcb4E5gY8J2FNrcba67z0wY+9/nbc+IACDcmgUZw93/B6hP2r0AeCJ4/gRw/cms08ng7jXu/qfgeSPxD4YJZHjbj7GuRka328xKgGuAxxN2Z3Sbe9Dnbc+UADjaegRRMsbdayD+QQmMTnN9+pWZlQLnA38gAm0PukLWAbXAimBdjUxv98PAXUBXwr5Mb3M3B142szXBmijQD23PlEXhw6xZIBnCzIYAPwe+4u77zFL9z59Z3L0TmBnMrvt8wroaGcnMrgVq3X2NmX0yzdVJhznuXm1mo4EVZrapxzN6IVOuAMKsWZDpdprZOIDgsTbN9ekXZpZD/MP/aXfvXk8iEm2HI9bVyOR2zwGuM7MK4l26l5rZU2R2mw9y9+rgsRZ4nng3d5+3PVMCIMyaBZluKfFFdggeX0xjXfqFxb/q/xew0d0fSjiU0W0/xroaGdtud7/H3UvcvZT4f8+vuvtNZHCbu5lZgZkVdj8HrgTW0w9tz5hfApvZ1cT7DLvXLPi39Nao/5jZT4FPEp8edidwL/AC8DNgElAJfNrdk28UD2hmdjHwv8A7HOoX/n/E7wNkbNvN7FziN/0S19X4tpmNJIPb3S3oAvqau18bhTab2UeIf+uHeDf9M+7+b/3R9owJABEROT6Z0gUkIiLHSQEgIhJRCgARkYhSAIiIRJQCQEQkohQAIimYWZGZfTHd9RDpTwoAkdSKgCMCIJh5ViQjKABEUvsucHowH/ubwToEzwDvBBOzfS/Y/7aZfb77JDP7p4T93fP2F5jZr4L5/Neb2d+kq1EiiTJlMjiRvnY3cLa7zwx+ifqrYPvDYHbGBne/0MzygDfM7GVgSvA3m/gEhUvN7BNAMVDt7tcAmNmwk98ckSMpAETC+aO7fxg8vxI418z+KtgeRvyD/8rgb22wf0iw/3+BB83sAeAld//fk1dtkaNTAIiE05Tw3IAvufvyxAJmdhXw7+7+o+STzewC4Grg383sZXf/dr/WViQE3QMQSa0RKDzKseXAF4KpqTGzqcGsjcuBzwXrFWBmE8xstJmNB5rd/SngQWBW/1dfpGe6AhBJwd3rzOwNM1sPHCA+62q3x4FS4E/BFNW7gOvd/WUzOwv4fbBIzX7gJuAM4Htm1gW0A184eS0ROTrNBioiElHqAhIRiSgFgIhIRCkAREQiSgEgIhJRCgARkYhSAIiIRJQCQEQkov4//Z10GuFcfSsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot test curve\n",
    "RF_results.plot(x='trees', y='mean_te')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>trees</th>\n",
       "      <th>mean_tr</th>\n",
       "      <th>mean_te</th>\n",
       "      <th>std_te</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>20</td>\n",
       "      <td>0.058517</td>\n",
       "      <td>0.054151</td>\n",
       "      <td>0.000641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>30</td>\n",
       "      <td>0.057540</td>\n",
       "      <td>0.053590</td>\n",
       "      <td>0.000666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>40</td>\n",
       "      <td>0.056998</td>\n",
       "      <td>0.053314</td>\n",
       "      <td>0.000734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>50</td>\n",
       "      <td>0.056662</td>\n",
       "      <td>0.053034</td>\n",
       "      <td>0.000823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>21</td>\n",
       "      <td>0.055013</td>\n",
       "      <td>0.051345</td>\n",
       "      <td>0.000606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>19</td>\n",
       "      <td>0.054952</td>\n",
       "      <td>0.051217</td>\n",
       "      <td>0.000653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>17</td>\n",
       "      <td>0.054911</td>\n",
       "      <td>0.051216</td>\n",
       "      <td>0.000696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>15</td>\n",
       "      <td>0.054788</td>\n",
       "      <td>0.051041</td>\n",
       "      <td>0.000708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>13</td>\n",
       "      <td>0.054550</td>\n",
       "      <td>0.050749</td>\n",
       "      <td>0.000694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>11</td>\n",
       "      <td>0.054385</td>\n",
       "      <td>0.050429</td>\n",
       "      <td>0.000721</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   trees   mean_tr   mean_te    std_te\n",
       "11    20  0.058517  0.054151  0.000641\n",
       "12    30  0.057540  0.053590  0.000666\n",
       "13    40  0.056998  0.053314  0.000734\n",
       "14    50  0.056662  0.053034  0.000823\n",
       "10    21  0.055013  0.051345  0.000606\n",
       "9     19  0.054952  0.051217  0.000653\n",
       "8     17  0.054911  0.051216  0.000696\n",
       "7     15  0.054788  0.051041  0.000708\n",
       "6     13  0.054550  0.050749  0.000694\n",
       "5     11  0.054385  0.050429  0.000721"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ten best combinations according to the mean test score\n",
    "RF_results.sort_values(by='mean_te', ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 5.49%\n"
     ]
    }
   ],
   "source": [
    "# Report test score\n",
    "acc_RF = 100*grid_RF_cv.score(X_te, y_te)\n",
    "print('Test accuracy: {:.2f}%'.format(acc_RF))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = grid_RF_cv.predict(X_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "tn, fp, fn, tp = confusion_matrix(y_te, y_pred).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True positive:  5746\n",
      "True negative:  638287\n",
      "False positive:  98874\n",
      "False negative:  322\n",
      "Positive pixels:  6068\n",
      "Negative pixels:  737161\n"
     ]
    }
   ],
   "source": [
    "print('True positive: ', tp)\n",
    "print('True negative: ', tn)\n",
    "print('False positive: ', fp)\n",
    "print('False negative: ', fn)\n",
    "print('Positive pixels: ', y_te[y_te == 1].size)\n",
    "print('Negative pixels: ', y_te[y_te == 0].size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAEWCAYAAABLzQ1kAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAsSElEQVR4nO3deZyXVd3/8dd7hlVklUVkUxM1NDXBLcswC7BFrVuL0iSjzDLtNs2f1l2Q3lR255ppbuSSpmZqlJmRyu1yuwAuuSLkiiDbsCMwy+f3x3UGvjPO8h2cYS5m3s/H43rMdZ3rnHOd76Cf75lznetcigjMzCx/Slq7AWZmVjcHaDOznHKANjPLKQdoM7OccoA2M8spB2gzs5xygLb3RVJXSX+RtFLSH99HPcdL+kdztq21SPqYpDmt3Q7b9snzoNsHSV8Bvg/sCawGngGmRMQj77PerwKnAR+JiIr32868kxTA8IiY19ptsbbPPeh2QNL3gUuAnwEDgKHAFcDRzVD9MOCV9hCciyGpQ2u3wdqQiPDWhjegJ7AGOK6BPJ3JAviCtF0CdE7nRgPzgTOBxcBC4KR07qfARqA8XWMiMBn4fUHdOwMBdEjHXwNeJevFvwYcX5D+SEG5jwAzgZXp50cKzs0AzgceTfX8A+hbz2erbv/ZBe0/Bvg08ApQBvywIP+BwGPAipT3cqBTOvdQ+ixr0+f9UkH9/w94B7ipOi2V+UC6xv7peCdgKTC6tf/b8Jb/zT3otu8QoAtwVwN5fgQcDOwH7EsWpP6r4PyOZIF+EFkQ/o2k3hExiaxXfltEbB8R1zXUEEndgMuAIyOiO1kQfqaOfH2Ae1LeHYCLgHsk7VCQ7SvASUB/oBNwVgOX3pHsdzAI+AlwDXACMBL4GPATSbumvJXAGUBfst/dEcB3ACLisJRn3/R5byuovw/ZXxMnF144Iv5NFrxvlrQd8Dvg+oiY0UB7zQAPcbQHOwBLo+EhiOOB8yJicUQsIesZf7XgfHk6Xx4RfyPrPe6xhe2pAvaW1DUiFkbEC3Xk+QwwNyJuioiKiPgD8DLwuYI8v4uIVyLiXeB2si+X+pSTjbeXA7eSBd9LI2J1uv4LwD4AETE7Ih5P130duAr4eBGfaVJEbEjtqSEirgHmAk8AA8m+EM0a5QDd9i0D+jYyNroT8EbB8RspbVMdtQL8OmD7pjYkItaSDQucAiyUdI+kPYtoT3WbBhUcv9OE9iyLiMq0Xx1AFxWcf7e6vKTdJf1V0juSVpH9hdC3gboBlkTE+kbyXAPsDfw6IjY0ktcMcIBuDx4D1pONu9ZnAdmf59WGprQtsRbYruB4x8KTEXFfRHyKrCf5Mlngaqw91W16ewvb1BRXkrVreET0AH4IqJEyDU6FkrQ92bj+dcDkNIRj1igH6DYuIlaSjbv+RtIxkraT1FHSkZJ+mbL9AfgvSf0k9U35f7+Fl3wGOEzSUEk9gXOrT0gaIOmoNBa9gWyopLKOOv4G7C7pK5I6SPoSMAL46xa2qSm6A6uANal3/+1a5xcBu76nVMMuBWZHxDfIxtZ/+75bae2CA3Q7EBEXkc2B/i9gCfAW8F3g7pTlv4FZwL+A54CnUtqWXGs6cFuqazY1g2oJ2WyQBWQzGz5OugFXq45lwGdT3mVkMzA+GxFLt6RNTXQW2Q3I1WS9+9tqnZ8M3CBphaQvNlaZpKOBcWTDOpD9O+wv6fhma7G1WX5Qxcwsp9yDNjPLKQdoM7OccoA2M8spB2gzs5xq1wu79O1TGjsP6djazbAmeGWepxBva1avW7g0Ivptafmxh3eLZWV1zcZ8r9n/2nBfRIzb0mvlTbsO0DsP6ciT9w1p7WZYE4w76oTWboI10fSZk2s/Fdoky8oqefK+oUXlLR04t7GnPrcp7TpAm1n+BVBFVWs3o1U4QJtZrgVBeRQ3xNHWOECbWe65B21mlkNBUNlOn3h2gDaz3KtqeMHANssB2sxyLYBKB2gzs3xyD9rMLIcCKG+nY9B+1NvMci0IKovciiGpl6Q7JL0s6SVJh0jqI2m6pLnpZ++C/OdKmidpjqSxBekjJT2Xzl0mSSm9s6TbUvoTknYuKDMhXWOupAmNtdUB2szyLaCyyK1IlwJ/j4g9yd5i/xJwDnB/RAwH7k/HSBoBjAf2InvxwhWSSlM9V5K9xX142qofMZ8ILI+I3YCLgQtSXX2AScBBwIHApMIvgro4QJtZrmVPEha3NUZSD+AwsvdDEhEbI2IFcDRwQ8p2A5vf4Xk0cGt6Y/trwDzgQEkDgR4R8Vhkbz25sVaZ6rruAI5IveuxwPSIKIuI5cB0Ngf1OjlAm1nOicoiN7I32M8q2E6uVdmuZK99+52kpyVdm96ROSAiFgKkn/1T/kFkr4irNj+lDUr7tdNrlImICmAlsEMDddXLNwnNLNeym4SNvVh9k6URMaqB8x2A/YHTIuIJSZeShjPqUdeFo4H0LS1TJ/egzSzXsnnQRfegGzMfmB8RT6TjO8gC9qI0bEH6ubggf+GSl4PJXno8P+3XTq9RRlIHoCfZS5Lrq6teDtBmlntVoaK2xkTEO8BbkvZISUcALwLTgOpZFROAP6f9acD4NDNjF7KbgU+mYZDVkg5O48sn1ipTXdexwANpnPo+YIyk3unm4JiUVi8PcZhZrlX3oJvRacDNkjoBrwInkXVWb5c0EXgTOA4gIl6QdDtZEK8ATo3YtLTet4Hrga7AvWmD7AbkTZLmkfWcx6e6yiSdD8xM+c6LiLKGGuoAbWa5FojKZvxjPyKeAeoapz6invxTgCl1pM8C9q4jfT0pwNdxbiowtdi2OkCbWe4VM3zRFjlAm1muBWJjlDaesQ1ygDazXMseVGmf8xkcoM0s95r5JuE2wwHazHItQlSGe9BmZrlU5R60mVn+ZDcJ22eoap+f2sy2Gb5JaGaWY5WeB21mlj/N/SThtsQB2sxyr8qzOMzM8idbLMkB2swsdwJR7ke9zczyJwI/qGJmlk/ygypmZnkUuAdtZpZbvkloZpZDQXHvG2yLHKDNLNcCKPdaHGZmeSSvB21mlkeBnyQ0M8st96DNzHIoQu5Bm5nlUXaT0I96m5nlUPt9J2H7/NRmts3IbhKqqK0Ykl6X9JykZyTNSml9JE2XNDf97F2Q/1xJ8yTNkTS2IH1kqmeepMskKaV3lnRbSn9C0s4FZSaka8yVNKGxtjpAm1nuVVJS1NYEh0fEfhExKh2fA9wfEcOB+9MxkkYA44G9gHHAFZKqx1uuBE4GhqdtXEqfCCyPiN2Ai4ELUl19gEnAQcCBwKTCL4K6OECbWa5VP0nYXD3oehwN3JD2bwCOKUi/NSI2RMRrwDzgQEkDgR4R8VhEBHBjrTLVdd0BHJF612OB6RFRFhHLgelsDup1coA2s9yroqSoDegraVbBdnId1QXwD0mzC84PiIiFAOln/5Q+CHiroOz8lDYo7ddOr1EmIiqAlcAODdRVL98kNLNci4DyqqL7kksLhi3qc2hELJDUH5gu6eUG8tbVLY8G0re0TJ3cgzazXMuGOEqK2oqqL2JB+rkYuItsPHhRGrYg/Vycss8HhhQUHwwsSOmD60ivUUZSB6AnUNZAXfVygDaz3KtM63E0tjVGUjdJ3av3gTHA88A0oHpWxQTgz2l/GjA+zczYhexm4JNpGGS1pIPT+PKJtcpU13Us8EAap74PGCOpd7o5OCal1ctDHDmzZmUpF581hNdf7oIE37/oTWY+0IPH7uuJBL36lnPWJW+yw44VVJTDxWcNZd5zXamsEJ88rozxpy1m/Tox5Vs7s+D1zpSUBgd/ahUTf7QQgMXzO/I//zmUtStLqaoSX//hAg48YjXPPLo9V03aPBz21r8788Mr3uAjR65srV/FNunoz73MkWPmIcG9/9iNu6ftya67lHHad56kU8cqKivF5b89gFfm9qW0tIr/PO1xdtu1jNLS4P4Hd+G2O/ama9dyfvXz6Zvq7Nt3HQ/M2Jmrrt38l/tHP/Im/3XOw5z2/XHMnbdDa3zUraZ6ml0zGQDclWbEdQBuiYi/S5oJ3C5pIvAmcBxARLwg6XbgRaACODUiKlNd3wauB7oC96YN4DrgJknzyHrO41NdZZLOB2amfOdFRFlDjW2xAC0pgIsi4sx0fBawfURMLrL8ZGBNRPyqCdf8G/CViFghaU1EbN/0lreuK38yiFGjV/Hja16nfKPY8G4Jw/ZYzISz3wHg7mv78vuLd+R7F8znob/0onyDuOqBOaxfJ04e/UFGH7OCXjuU8x+nLGG/Q9dQvlH8vy9+gJkPdOeAT6zmlksHcNjnVvC5Cct445XO/PiED3Djky+y36FruPKfcwBYtbyUkw79IPt/fFVr/iq2OcOGruDIMfP43pnjKK8oYcrkB3ly5k5M/NrT3PyHDzHrqUEcMPJtvvG1pzn7R5/iY4e+QccOVXz79M/SuVMFV//mr8x4aGcWLd6eU//z05vq/fVF9/LoY5v/Mu7atZyjP/cyL81p24F5s+Z71DsiXgX2rSN9GXBEPWWmAFPqSJ8F7F1H+npSgK/j3FRgarHtbckhjg3AFyT1bcFr1BARn46IFVvres1t7eoSnnu8G+O+kn2pduwUbN+zkm7dqzblWf9uCUqdCQnWryuhsgI2ri+hQ6cqttu+ki7bBfsdumZTHcM/9C5LFnbcVGbd6mwa59pVpfQZUP6edjxyTy8OOHwVXbZr8P6F1TJ0yEpentOXDRs7UFVVwnMv9Ocjh7wFIbbbLvs9d+tWzrKyrqmE6NKlgpKSKjp1rqS8ooS16zrWqHOngavo1XM9z7/Qf1Paicc/yx//NILyje3n8eeq9F7Cxra2piUDdAVwNXBG7ROShkm6X9K/0s+h9dQxQtIMSa9KOr2g/N1piswLhdNo0hNCW+0Lobm980Zneu5QwYVnDOU7n9qdi88cwvp12T/R736xI8ePHMEDd/bmxB9kwxUf++wKumxXxZf325sTDhjBsacsoUfvyhp1rllZyuPTe/Dhj2YB+4Qz3+GBO3tz/MgR/Piru3LqlPnUNuPPvRh9zIqW/bBt0Otv9GLvvRbTvfsGOneq4ICRC+jXdx2/vXYk3zjpaW667i6+cdJT/O7G/QB4+NGhrF/fgVtuuJObrruLP939Qdas6VyjztGHvcH/PjKM6gkAH9i1jH591/HkrMG0F9ksjtKitrampW8S/gY4XlLPWumXAzdGxD7AzcBl9ZTfk2xyd/VTN9Xdi69HxEhgFHC6pKL/1pN0cvUcySXLKhsvsBVVVsK857bjsycu5Yrpr9BluypuuzzrOZ10zjvcPPtFPvGF5Uyb2g+AOU93o6Q0uOXp57nxiZf402/7sfCNTpvrq4Cff2cYR09cysBhGwGYcXdvPvXFMm6e/SLn3/QqvzxtGFWbO+gsW9SB11/qyqjRHt5oqrfm9+SPd47g5+fdz3//9AFefa0XlZUlfPbIuVx17Ui+OvHzXHXtSM447QkA9th9KVVV4vivfYEJ3zyG/zj6JXYcsLpGnR//2OvMeGgYAFLwrYmzuWbq/lv9s7WmrfSgSi61aICOiFVkT9icXuvUIcAtaf8m4KP1VHFPeoJnKdm0lwEp/XRJzwKPk01bGd6ENl0dEaMiYlS/HfL1jdt3YDn9Bpaz5/7rAPjoZ1cw77muNfIc/vnlPPK37Pvuwbt6Merw1XToCL36VjDigLW88ux2m/Je8oMhDNplA1/45pJNaX//Qx8O+9wKAEaMWsfGDWJV2eZbEQ/9pRcfOXIFHWr+pW1Fum/6bnz3jE/zg3PHsHpNZxYs6M4nP/HqpjHkhx8dyu67LwXg8MNeZ/ZTA6msLGHlyi688HI/hu+2+Z7RLjsvp7Q0mPfvrP/RtWs5w4at5JdT/skN19zNnnssZfKP/pfhuy3b+h90K/MQR8u5hOzZ9G4N5KlvsHNDwX4l0EHSaOCTwCERsS/wNNDlfbcyB/r0r6DvTht5a172Z+4zD3dn6PANvP3q5l7x4/f1ZMhu2a+l36BynnlkeyKyseiXn+rGkN3WA3D9BTuydnUpp5z3do1r9B9UzjOPdAfgzbmd2bihhJ47VGw6P+Pu3h7eeB969sx+//36ruXQQ95ixkPDWFbWlX32zqbV7rfPIhYs6AHA4iXd2HefRUDQuXMFe+6+lPlv99hU1+jDXmfGQztvOl63rhNfOuFYJnzzGCZ88xhentOXyVM+3m5mcbTHHnSLT7NLU0tuJwvS1Xcv/49s6slNwPHAI02osifZQiTrJO0JHNyc7W1tp/7321zw3WFUlIsdh27kzIvf5OKzhjD/350pKYH+gzZy+gXZuPFRJy3lwjOGcvLhe0CIMV9axq4j1rNkQUf+cOmODNltPaeO2SPlXcKRx5dx8qS3ueSsIdx5TT8EnHXxm5tuOr7zVieWLOjIPoesaaVPv+378TkP0b37BiorS/jNbw9gzdrOXHr5QZzyzdmUllaxcWMpl/7mQAD+8rfdOfN7j3PV5fcAwfT7P8Brr29eO+ewj77Jj386ulU+R9601wX7lc2fboGKC6a5SRoAvAb8MiImp+X3pgJ9gSXASRHxZq3ykymYZifpeeCzwELgbrJn2OcA/YDJETFD0uvAqIhYWsw0u1H7dokn7xvSUBbLmXFHndDaTbAmmj5z8uwiHr+uV+89+8cnph5bVN47D73yfV0rb1qsB10YHCNiEbBdwfHrwCcaKT+51nHhfMMj6ymzc13XN7NtW1scviiGnyQ0s1xr5icJtykO0GaWew7QZmY5VD0Puj1ygDaz3GuLc5yL4QBtZrkWARXFL9jfpjhAm1nueYjDzCyHPAZtZpZj4QBtZpZPvkloZpZDER6DNjPLKVHpWRxmZvnkMWgzsxzyWhxmZnkV2Th0e+QAbWa551kcZmY5FL5JaGaWX+11iKN9fi2Z2TYlQkVtxZJUKulpSX9Nx30kTZc0N/3sXZD3XEnzJM2RNLYgfaSk59K5y6Ts7Z6SOku6LaU/kV7xV11mQrrGXEkTGmunA7SZ5VpE8wdo4HvASwXH5wD3R8Rw4P50jKQRZC+43gsYB1whqTSVuRI4GRietnEpfSLZi613Ay4GLkh19QEmAQcBBwKTCr8I6uIAbWa5VxUqaiuGpMHAZ4BrC5KPBm5I+zcAxxSk3xoRGyLiNWAecKCkgUCPiHgssjdv31irTHVddwBHpN71WGB6RJRFxHJgOpuDep0coM0s9yKK24p0CXA2UFWQNiAiFmbXioVA/5Q+CHirIN/8lDYo7ddOr1EmIiqAlcAODdRVLwdoM8u1QFRVlRS1AX0lzSrYTi6sS9JngcURMbvIy9fVLY8G0re0TJ08i8PMcq8JkziWRsSoBs4fChwl6dNAF6CHpN8DiyQNjIiFafhicco/HxhSUH4wsCClD64jvbDMfEkdgJ5AWUofXavMjIY+jHvQZpZvzXiTMCLOjYjBEbEz2c2/ByLiBGAaUD2rYgLw57Q/DRifZmbsQnYz8Mk0DLJa0sFpfPnEWmWq6zo2XSOA+4Axknqnm4NjUlq93IM2s/xr+XnQvwBulzQReBM4DiAiXpB0O/AiUAGcGhGVqcy3geuBrsC9aQO4DrhJ0jyynvP4VFeZpPOBmSnfeRFR1lCjHKDNLPdaYjW7iJhBGmKIiGXAEfXkmwJMqSN9FrB3HenrSQG+jnNTganFtrHeAC3p1zTwvRURpxd7ETOzLRVAVZXX4qht1lZrhZlZfQLwcqM1RcQNhceSukXE2pZvkplZTV6Lox6SDpH0IumxSEn7SrqixVtmZlYtitzamGKm2V1C9ojiMoCIeBY4rAXbZGZWoLgpdm3xtVhFzeKIiLfSQk3VKuvLa2bW7Npg77gYxQTotyR9BAhJnYDTqbkKlJlZywmIdjqLo5ghjlOAU8kW9Xgb2C8dm5ltJSpya1sa7UFHxFLg+K3QFjOzurXTIY5iZnHsKukvkpZIWizpz5J23RqNMzMDPIujAbcAtwMDgZ2APwJ/aMlGmZltUv2gSjFbG1NMgFZE3BQRFWn7PW3yu8rM8qqZF+zfZjS0FkeftPugpHOAW8kC85eAe7ZC28zMMu10FkdDNwlnU/MtAN8qOBfA+S3VKDOzQmqDveNiNLQWxy5bsyFmZnVqozcAi1HUk4SS9gZGkL0iBoCIuLGlGmVmtlnbvAFYjEYDtKRJZO/RGgH8DTgSeITsNeNmZi2vnfagi5nFcSzZmwbeiYiTgH2Bzi3aKjOzQlVFbm1MMUMc70ZElaQKST3I3nbrB1XMbOvwgv0NmiWpF3AN2cyONcCTLdkoM7NCnsVRj4j4Ttr9raS/Az0i4l8t2ywzswIO0DVJ2r+hcxHxVMs0yczMoOEe9IUNnAvgE83clq3ulX9tx9id9mvtZlhTlHgp8vbIQxy1RMThW7MhZmZ1Cvyot5lZbrXTHnQx86DNzFqVorit0XqkLpKelPSspBck/TSl95E0XdLc9LN3QZlzJc2TNEfS2IL0kZKeS+cuU3pxq6TOkm5L6U9I2rmgzIR0jbmSJjTWXgdoM8u/5luwfwPwiYjYl+z1feMkHQycA9wfEcOB+9MxkkYA44G9gHHAFZJKU11XAicDw9M2LqVPBJZHxG7AxcAFqa4+wCTgIOBAYFLhF0FdinmjiiSdIOkn6XiopAOL+EWYmTWPZgrQkVmTDjumLYCjgRtS+g3AMWn/aODWiNgQEa8B84ADJQ0km3L8WEQE2dIXhWWq67oDOCL1rscC0yOiLCKWA9PZHNTrVEwP+grgEODL6Xg18JsiypmZvW/FDm8UO9NDUqmkZ8ieip4eEU8AAyJiIUD62T9lHwS8VVB8fkoblPZrp9coExEVwEpghwbqqlcxNwkPioj9JT2dLrhcUqciypmZNY/iZ3H0lTSr4PjqiLi6MENEVAL7pSek70qrddanrgtHA+lbWqZOxQTo8jTmEgCS+tEmlyUxs7xqwjzopRExqpiMEbFC0gyyYYZFkgZGxMI0fLE4ZZsPDCkoNhhYkNIH15FeWGa+pA5AT6AspY+uVWZGQ20sZojjMuAuoL+kKWRLjf6siHJmZs2jmcagJfVLPWckdQU+CbwMTAOqZ1VMAP6c9qcB49PMjF3IbgY+mYZBVks6OI0vn1irTHVdxwIPpHHq+4Axknqnm4NjUlq9ilmL42ZJs8mWHBVwTET4cS4z2zqaML5chIHADWlUoAS4PSL+Kukx4HZJE4E3geMAIuIFSbcDLwIVwKlpiATg28D1QFfg3rQBXAfcJGkeWc95fKqrTNL5wMyU77yIKGuoscUs2D8UWAf8pTAtIt5srKyZWbNopgCdFnr7cB3py8g6oXWVmQJMqSN9FvCe8euIWE8K8HWcmwpMLba9xYxB38PmAe4uwC7AHLJ5gWZmLU7t9K5XMUMcHyo8Tqvcfaue7GZm1kyavBZHRDwl6YCWaIyZWZ3a6VocxYxBf7/gsATYH1jSYi0yMyvUvDcJtynF9KC7F+xXkI1J/6llmmNmVgcH6PdKU1G2j4gfbKX2mJm9lwN0TZI6RERFQ6++MjNracKzOOryJNl48zOSpgF/BNZWn4yIO1u4bWZmHoNuRB9gGdk7CKvnQwfgAG1mW4cD9Hv0TzM4nue9KzG101+XmbWKdhpxGgrQpcD2bMESeWZmzclDHO+1MCLO22otMTOrjwP0e7TP95ybWb6EZ3HUpc6VnczMtjr3oGtqbJ1SM7OtxWPQZmZ55QBtZpZDRb7Oqi1ygDazXBMe4jAzyy0HaDOzvHKANjPLKQdoM7Mc8mp2ZmY55gBtZpZPftTbzCynPMRhZpZH7fhBlZLWboCZWaOiyK0RkoZIelDSS5JekPS9lN5H0nRJc9PP3gVlzpU0T9IcSWML0kdKei6du0ySUnpnSbel9Cck7VxQZkK6xlxJExprrwO0meVa9ZOExWxFqADOjIgPAgcDp0oaAZwD3B8Rw4H70zHp3HhgL2AccIWk0lTXlcDJwPC0jUvpE4HlEbEbcDFwQaqrDzAJOAg4EJhU+EVQFwdoM8s9VUVRW2MiYmFEPJX2VwMvAYOAo4EbUrYbgGPS/tHArRGxISJeA+YBB0oaCPSIiMciIoAba5WprusO4IjUux4LTI+IsohYDkxnc1Cvk8egzSzfmjYG3VfSrILjqyPi6roypqGHDwNPAAMiYiFkQVxS/5RtEPB4QbH5Ka087ddOry7zVqqrQtJKYIfC9DrK1MkB2sxyrwmzOJZGxKhG65O2B/4E/GdErErDx3VmrSOt9ku0C9O3tEydPMRhZvnXTDcJASR1JAvON0fEnSl5URq2IP1cnNLnA0MKig8GFqT0wXWk1ygjqQPQEyhroK56OUCbWe41103CNBZ8HfBSRFxUcGoaUD2rYgLw54L08Wlmxi5kNwOfTMMhqyUdnOo8sVaZ6rqOBR5I49T3AWMk9U43B8ektHp5iMPM8q/55kEfCnwVeE7SMynth8AvgNslTQTeBI4DiIgXJN0OvEg2A+TUiKhM5b4NXA90Be5NG2RfADdJmkfWcx6f6iqTdD4wM+U7r7FXCzpAm1m+NeNbvSPiEeoeC4Z6XpQdEVOAKXWkzwL2riN9PSnA13FuKjC12PY6QJtZrvmNKmZmeRbtM0I7QJtZ7rkHbduMjp2ruPDOeXTsFJR2CB6+pxc3/WpHvvHjBRz8qVWUbxQL3+jEhWcMZe2qUvY/bDVf/+FCOnQMKsrFNecP5NlHu7f2x2jzbnjsed5dW0JVpaisEKd9Zk9+eMWrDP7ABgC69ahk7apSvjP2g5vK9NtpI9c8+CK/v2ggd1w1AIAOHas49b/fYp9D1hBVcP0vd+KRvzX4hHDb0o4XS8ptgJa0JiK2Lzj+GjAqIr7bQJmjgBER8QtJk4E1EfGrFm/sVla+QZx93AdYv66U0g7BRXfPY+YD3Xnqoe5M/dlAqirFxB8tYPxpi7huyk6sLCvlJxN2oWxRR4bt8S4/u+VVjh+5V2t/jHbh7ON2Z9Xyzf+b/ew7u27aP/nH81m7urRG/lMmz2fmgz1qpH359HdYsbQjEw/bCyno3quS9qa9rgfdpuZBR8S0iPhFa7ej5Yn167L/sTt0DEo7BhHw1P92p6oyu0H90uxu9B1YDsC/n9+OskUdAXhjThc6dQ46dmqn/8XnRnDY55bz4J8394QPGbuChW924o1XutTIOfZLy7j18qw3HaEaAb+9UFVxW1uzTQZoSZ9Ly/g9Lemfkgak9K9Jury127c1lJQEV0yfw23/eoGnH9qeOU93q3F+7JfLmPlAj/eU++hnVvLvF7pSvnGb/KfftgT87Ja5XP63lzjy+KU1Tu190BqWL+nIgteyYNy5ayVf/M4ifn/RwBr5uvWoAGDCDxZy+b0v8aPfvkqvvuVbp/15EWQ3CYvZ2pg8/1/aVdIz1RtwXsG5R4CDI+LDwK3A2cVWKulkSbMkzSpnQ/O2eCuqqhLf+dQeHD9yBHvst45he7y76dyXT19EZQU8cGevGmWG7b6eiT9ayKVnD8Za3hmf353vHvlBfvTV3ThqwhL2Pmj1pnOHH72cGQW95xPPXMhd1/Tf9JdRtdJS6LdTOS/O6sZ3j/wgL83uxjd//PZW+wx50YzLjW5T8vy30rsRsV/1QfUYdDocDNyWnpnvBLxWbKVpZaurAXqozzb/T7p2VSnPPrY9Bxy+mjfmdOWTx5Vx4CdXcc6XPkDhfPy+Azfyk+te43++N5SFb3RuvQa3I2WLOgGwcllHHv17T/bcbx3PP9GdktLg0CNX8N1P77kp754fXstHP7OCiT96m+17VBIBGzeIadf3Y/26Eh69txcAD/+1N+PGL2uNj9O6tvn/U7dMngN0Q34NXBQR0ySNBia3amu2sp59KqioEGtXldKpSxX7f2wNt/+mP6NGr+KLpy7mB1/YjQ3vbv7jqFuPSs6/8TV+9/OBvDizWwM1W3Pp3LWSkhJ4d20pnbtWMvKw1dx8yY4A7P+xVbz17y4sXdhpU/4z/2OPTfsnfH8B69eWMu36bMXLx6f3ZJ9D1vDs/3Vnv4+u5o25Nceo2zo/qLLt6QlU/53X6Gtj2po+A8o569I3KSmBkhJ46C89eeKfPfjdoy/RsXPw89v+DcDLs7tx2TmDOeqkpey0y0a+csYivnLGIgDOHb8rK5d1bM2P0ab17lfBpGtfBaC0NHjw7t7MmtETgI8ftZwZdxc/Te66n+3E2Ze+wSk/fYuVyzpy4feHtUibcyuKW4y/LVLkdGC9oWl2ko4me5XM22SLaR8QEaNr5ZlMI9PseqhPHKQ6H7+3vCopbTyP5co/K2+bXcwazfXp3mtwfPiw7xWV9+G/nP2+rpU3ue1BFwbndHw92cpRRMSf2by0X315JrdsC81sa/EQh5lZHgXQToc4HKDNLP/aZ3x2gDaz/PMQh5lZTrXXWRwO0GaWb17Nzswsn7IHVdpnhHaANrP8a4Mr1RXDAdrMcs89aDOzPPIYtJlZXrXftTgcoM0s/zzEYWaWQ9E2X2dVjDy/UcXMLNNMr7ySNFXSYknPF6T1kTRd0tz0s3fBuXMlzZM0R9LYgvSRkp5L5y6TpJTeWdJtKf0JSTsXlJmQrjFXUlHLJDtAm1n+RZFb464HxtVKOwe4PyKGA/enYySNAMYDe6UyV0iqXu/2SuBkYHjaquucCCyPiN3IlkS+INXVB5gEHAQcCEwq/CKojwO0meWeqqqK2hoTEQ8BZbWSjwZuSPs3AMcUpN8aERsi4jVgHnBgetVej4h4LLIF9W+sVaa6rjuAI1LveiwwPSLKImI5MJ33flG8h8egzSzfgqY8qNJX0qyC46vTe0gbMiAiFgJExEJJ/VP6ILIXglSbn9LK037t9Ooyb6W6KiStBHYoTK+jTL0coM0s10Q05UGVpc34RhXVkRYNpG9pmXp5iMPM8q+ZbhLWY1EatiD9XJzS5wNDCvINBhak9MF1pNcoI6kD2ftTyxqoq0EO0GaWfy0boKex+eXTE9j8Or1pwPg0M2MXspuBT6bhkNWSDk7jyyfWKlNd17HAA2mc+j5gjKTe6ebgmJTWIA9xmFm+NW0MukGS/gCMJhurnk82s+IXwO2SJgJvAscBRMQLkm4HXgQqgFMjojJV9W2yGSFdgXvTBnAdcJOkeWQ95/GprjJJ5wMzU77zIqL2zcr3cIA2s9wrZoZGMSLiy/WcOqKe/FOAKXWkzwL2riN9PSnA13FuKjC16MbiAG1mufe+hi+2aQ7QZpZvgQO0mVlutdO1OBygzSz3vGC/mVleOUCbmeVQBFS2zzEOB2gzyz/3oM3McsoB2swshwLwOwnNzPIoIDwGbWaWP4FvEpqZ5ZbHoM3McsoB2swsj7xYkplZPgXQTMuNbmscoM0s/9yDNjPLIz/qbWaWTwHhedBmZjnlJwnNzHLKY9BmZjkU4VkcZma55R60mVkeBVFZ2dqNaBUO0GaWb15u1MwsxzzNzswsfwII96DNzHIovGC/mVlutdebhIp2On0FQNIS4I3WbkcL6Assbe1GWJO05X+zYRHRb0sLS/o72e+nGEsjYtyWXitv2nWAbqskzYqIUa3dDiue/82sLiWt3QAzM6ubA7SZWU45QLdNV7d2A6zJ/G9m7+ExaDOznHIP2swspxygzcxyygE6hySFpAsLjs+SNLkJ5SdLOquJ1/ybpF5pf01TylrDav8+JX1N0uWNlDlK0jlpv8n/ntY2OEDn0wbgC5KKnZz/vkXEpyNixda6njUsIqZFxC9aux3Wuhyg86mC7K7+GbVPSBom6X5J/0o/h9ZTxwhJMyS9Kun0gvJ3S5ot6QVJJxekv741vxAsI+lzkp6Q9LSkf0oakNIb7WVb2+cAnV+/AY6X1LNW+uXAjRGxD3AzcFk95fcExgIHApMkdUzpX4+IkcAo4HRJOzR/062WrpKeqd6A8wrOPQIcHBEfBm4Fzm6NBlo+ebGknIqIVZJuBE4H3i04dQjwhbR/E/DLeqq4JyI2ABskLQYGAPPJgvLnU54hwHBgWXO332p4NyL2qz6Q9DWyL0iAwcBtkgYCnYDXtnrrLLfcg863S4CJQLcG8tQ3kX1DwX4l0EHSaOCTwCERsS/wNNDlfbfS3o9fA5dHxIeAb+F/DyvgAJ1jEVEG3E4WpKv9HzA+7R9P9idysXoCyyNinaQ9gYObpaH2fvQE3k77E1qzIZY/DtD5dyE1l1o8HThJ0r+ArwLfa0JdfyfrSf8LOB94vNlaaVtqMvBHSQ/TdpcbtS3kR73NzHLKPWgzs5xygDYzyykHaDOznHKANjPLKQdoM7OccoC2BkmqTI8oPy/pj5K2ex91XS/p2LR/raQRDeQdLekjW3CNOtcUKWatkaau4udV5qylOUBbY96NiP0iYm9gI3BK4UlJpVtSaUR8IyJebCDLaKDJAdqsLXGAtqZ4GNgt9W4flHQL8JykUkn/I2lmWmXvWwDKXC7pRUn3AP2rK0or7Y1K++MkPSXp2bRC385kXwRnpN77xyT1k/SndI2Zkg5NZXeQ9I+0GtxVgBr7EPWt6JfOXZjacr+kfintA5L+nso8nJ7CNGtxXizJiiKpA3Ak2dOIkK2St3dEvJaC3MqIOEBSZ+BRSf8APgzsAXyIbLGmF4GptertB1wDHJbq6hMRZZJ+C6yJiF+lfLcAF0fEI2mJ1fuADwKTgEci4jxJnwFqBNx6fD1doyswU9KfImIZ2ZonT0XEmZJ+kur+LtnSr6dExFxJBwFXAJ/Ygl+jWZM4QFtjuqYlMiHrQV9HNvTwZERUr7w2BtinenyZbH2J4cBhwB8iohJYIOmBOuo/GHiouq60/khdPkm2xnX1cQ9J3dM1vpDK3iNpeRGfqb4V/aqA21L674E7JW2fPu8fC67duYhrmL1vDtDWmBpLZQKkQLW2MAk4LSLuq5Xv09S/2l5h2WLWGyghW4WvcOnV6rYUvV5BrRX91kmaQf0ryEW67oravwOzrcFj0NYc7gO+Xf1SAEm7S+oGPASMT2PUA4HD6yj7GPBxSbuksn1S+mqge0G+f5ANN5Dy7Zd2HyJb1Q9JRwK9G2lrQyv6lQDVfwV8hWzoZBXwmqTj0jUkad9GrmHWLBygrTlcSza+/JSk54GryP46uwuYCzwHXAn8b+2CEbGEbNz4TknPsnmI4S/A56tvEpKt4jcq3YR8kc2zSX4KHCbpKbKhljcbaWtDK/qtBfaSNJtsjLn6zSfHAxNT+14Aji7id2L2vnk1OzOznHIP2swspxygzcxyygHazCynHKDNzHLKAdrMLKccoM3McsoB2swsp/4/z64FsT1u1cYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "disp = plot_confusion_matrix(grid_RF_cv, X_te, y_te, labels=[0, 1], display_labels=['No hail', 'Hail'], normalize=None)\n",
    "disp.ax_.set_title('Confusion matrix')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Probability of Detection (POD): 94.69347396176664\n",
      "False Alarm Rate (FAR): 13.412809413411724\n",
      "Positive Predictive Value (PPV): 0.05492257694513478\n"
     ]
    }
   ],
   "source": [
    "pod = 100*tp/(tp+fn)\n",
    "far = 100*fp/(fp+tn)\n",
    "ppv = tp/(tp+fp)\n",
    "print('Probability of Detection (POD):', pod)\n",
    "print('False Alarm Rate (FAR):', far)\n",
    "print('Positive Predictive Value (PPV):', ppv)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results are virtually the same as with the other scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_name, 'wb') as bfile: \n",
    "    s = pickle.dump(grid_RF_cv.best_estimator_, bfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_name, 'rb') as bfile:\n",
    "    rf_model_best = pickle.load(bfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get features importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = ['HRV_norm', 'HRV_norm_text', 'IR_108', 'IR_108_text', 'WV_062-IR_108', 'WV_062-IR_108_text']\n",
    "f_importance = rf_model_best['RF'].feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in rf_model_best['RF'].estimators_], axis=0)\n",
    "inds = np.argsort(f_importance)[::-1]\n",
    "f_importance_ordered = f_importance[inds]\n",
    "features_ordered = np.array(features)[inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcsAAAEICAYAAAAwft9dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgRElEQVR4nO3de5yVZb338c8XM1BUPFBbHFJK0dRIVPJYRi/ZeahEn8zTU0qadjLb5gnT3Sa1hLAks8cdWY9ZSltNiNIEbTsdBLRBOUmaIKighqfYEkoKv/3HfU0sx5m51sysmbXW8H2/XvNi3Yfrun/XWsqX617XWqOIwMzMzNrWp9oFmJmZ1TqHpZmZWYbD0szMLMNhaWZmluGwNDMzy3BYmpmZZTgszaxiJH1V0vXVrsOs0uTPWZrVBknLgX8B1pfs3j0inu5in5+JiHu6Vl39kTQO2C0iPlntWqz+eWZpVls+FhFblfx0OigrQdJbqnn9zqrXuq12OSzNapykAZJ+JOkZSSslXSFps3RsV0n/LekFSc9LuknStunYT4GdgV9JWiPpQkkjJa1o0f9ySaPS43GSbpP0M0n/A4xp7/qt1DpO0s/S4yGSQtKnJT0l6SVJn5P0PkkLJP1N0rUlbcdIuk/S9yStlvSIpMNLju8kabqkFyUtkXRmi+uW1v054KvAiWns89N5n5b0Z0kvS3pc0mdL+hgpaYWk8yStSuP9dMnxLSR9W9ITqb4/StoiHTtI0qw0pvmSRnbipbYa5rA0q30/AV4HdgP2BT4MfCYdE3AlsBOwJ/AOYBxARHwKeJKNs9VvlXm90cBtwLbATZnrl+NAYChwIjAJuAQYBewNnCDpgy3OfRwYCPwHcLuk7dOxKcCKNNbjgW+WhmmLun8EfBP4rzT2fdI5q4CPAtsAnwaulrRfSR87AgOABuAM4PuStkvHrgL2Bw4BtgcuBDZIagDuAK5I+88HfiHpbR14jqzGOSzNasu0NDv5m6Rpkv4FOAr4t4j4e0SsAq4GTgKIiCURcXdErIuI54DvAB9su/uyzI6IaRGxgSJU2rx+mS6PiFcjYibwd2BKRKyKiJXAHygCuNkqYFJEvBYR/wU8CnxE0juA9wMXpb7mAdcDn2qt7oh4pbVCIuKOiFgahd8BM4EPlJzyGnBZuv6dwBpgD0l9gNOBL0fEyohYHxGzImId8Engzoi4M137bqAJOLoDz5HVON/XN6stx5YuxpF0ALA58Iyk5t19gKfS8bcD11D8hb91OvZSF2t4quTxLu1dv0x/LXn8SivbW5Vsr4w3rjp8gmImuRPwYkS83OLYiDbqbpWkoyhmrLtTjGNLYGHJKS9ExOsl22tTfQOBfsDSVrrdBfiEpI+V7NscuDdXj9UPh6VZbXsKWAcMbPGXeLMrgQDeGxEvSDoWuLbkeMvl7n+nCAgA0nuPLW8XlrbJXb/SGiSpJDB3BqYDTwPbS9q6JDB3BlaWtG051jdsS+oL/AI4FfhlRLwmaRrFreyc54FXgV2B+S2OPQX8NCLOfFMr6zV8G9ashkXEMxS3Cr8taRtJfdKinuZbrVtT3Cr8W3rv7IIWXfwVeFfJ9l+AfpI+Imlz4FKgbxeuX2lvB86RtLmkT1C8D3tnRDwFzAKulNRP0nsp3lO8qZ2+/goMSbdQAd5KMdbngNfTLPPD5RSVbkn/GPhOWmi0maSDUwD/DPiYpCPS/n5psdDgjg/fapXD0qz2nUrxF/1iilustwGD0rGvA/sBqykWmdzeou2VwKXpPdDzI2I18AWK9/tWUsw0V9C+9q5fafdTLAZ6HvgGcHxEvJCOnQwMoZhlTgX+I70/2JZb058vSHowzUjPAW6hGMcpFLPWcp1Pccv2T8CLwASgTwry0RSrb5+jmGlegP9+7VX8pQRmVhMkjaH4AoX3V7sWs5b8Lx8zM7MMh6WZmVmGb8OamZlleGZpZmaW4c9Z9kIDBw6MIUOGVLsMM7O6Mnfu3OcjotWvKXRY9kJDhgyhqamp2mWYmdUVSU+0dcy3Yc3MzDIclmZmZhkOSzMzswyHpZmZWYbD0szMLMNhaWZmluGwNDMzy3BYmpmZZfhLCXqhhStXM2TsHdUuo64tH/+RapdgZjXEM0szM7MMh6WZmVmGw9LMzCzDYWlmZpbhsDQzM8twWJqZmWU4LM3MzDIclmZmZhkOSzMzswyHpZmZWYbD0szMLCMblpKulvRvJdszJF1fsv1tSSFpjxbtJkm6sJ1+L5a0RNKjko4o2f9WSZMl/UXSI5I+nvZ/RdJiSQsk/VbSLm30O07S+enxDZKWSZonab6kwzNjPTvVFJIGluyXpGvSsQWS9is5dq6khyUtkjRFUr92+h8jaaf2asjU99XOtjUzs84r54vUZwGfACZJ6gMMBLYpOX4IcBdwEvB1gHTe8cChrXUoaa90/t7ATsA9knaPiPXAJcCqiNg99bN9avYQMCIi1kr6PPAt4MQy6r8gIm6T9CFgMjC0nXPvA34NNLbYf1RqNxQ4ELgOOFBSA3AOsFdEvCLpljSuG9rofwywCHi6jLpb81Xgm51sa8mzN4/NnjNyzsTsOY2NjRWoxszqQTm3Ye+jCEQowm0R8LKk7ST1BfYELqIIiWaHAcsj4ok2+hwN/Dwi1kXEMmAJcEA6djpwJUBEbIiI59PjeyNibTpnDjC4nAGWmA00tHdCRDwUEcvbqPfGKMwBtpU0KB17C7CFpLcAW9JGEEo6HhgB3JRmultI2l/S7yTNTTP2QZIGpNn2HqndFElnShqfrjNP0k2t9H+WpCZJTevXri7vGTEzs7JkZ5YR8bSk1yXtTBGazaFzMLAaWBARCyRtkLRPRMynCM4p7XTbQBF4zVYADZK2TduXSxoJLAXOjoi/tmh/BvCbXO0tHAlM62CbZg3AUyXbK4CGiGiSdBXwJPAKMDMiZrbWQZrdng2cn9ptDnwPGB0Rz0k6EfhGRJyezrtB0neB7SLih1DcJo6I4W30P5li5kzfQUOjk+PcJOx4yvjsOY3+FV1mVqLcBT7Ns8vmsJxdsj0rnTMFOCnNsEYDt7bTn1rZFxThPRi4LyL2S9e56g0NpU9SzNDy98kKEyU9DvyMzt/CbLVeSdtRjPWdFLeT+6f6yrEH8B7gbknzgEtJs+WIuBtYCHwf+EwnazYzswopNyxnUQTjMIrbsHMoZpaHUAQpFGF5AjCKYra5qp3+VgDvKNkeTHH78gVgLTA17b8VKF1MM4riPc1jImJd2veNdGtyXhvXugDYjSKMflLGWDtS7yhgWUQ8FxGvAbez8ZZ1joCHI2J4+hkWER+Gf77nuyfFbHX79joxM7Pu15GZ5UeBFyNifUS8CGxLEZizASJiKUXYjaf9W7AA0ylmoX0lvZNi4cwDERHAr4CR6bzDgcUAkvYFfkARlP8M4oi4pDlw2rpYRGwAvgv0KV152wHTgVPTqtiDgNUR8QzF7deDJG0pSaneP7fTz8vA1unxo8DbJB2cxre5pL3TsXNTPycDP063bAFeK3lsZmY9pNywXEixCnZOi32rmxfgJFOAd7NxZtiqiHgYuIUiCO8CvphWwkKxWGicpAXAp4Dz0v6JwFbArWkmOb3M2puvGcAVQHsfZzlH0gqKmeMCbfyIzJ3A4xQLkX4IfCH1eT9wG/AgxfPRh/S+YRtuAP4zzYI3o1gxPEHSfGAecIik3SluvZ4XEX8Afk8xKyb1vaC1BT5mZtZ9VGSI9SZ9Bw2NQadNqnYZdW25F/iYbXIkzY2IEa0d8zf4mJmZZZTzpQSdlt4fnNBi97KIOK47r5sjaSrFCtZSF0XEjAr1/33e/IUM342I/1+J/s3MrGd1a1im8KlIAFVSd4d1RHyxO/s3M7Oe5duwZmZmGQ5LMzOzDIelmZlZhsPSzMwso1sX+Fh1DGsYQJM/J2hmVjGeWZqZmWU4LM3MzDIclmZmZhkOSzMzswyHpZmZWYZXw/ZCC1euZsjYO6pdhnWAf8uJWW3zzNLMzCzDYWlmZpbhsDQzM8twWJqZmWU4LM3MzDIclmZmZhkOSzMzswyHpZmZWYbD0szMLMNhaWZmluGwNDMzy+iVYSlpTfpziKRXJM2TtFjSjZI2b6fdDpLulbRG0rUtju0vaaGkJZKukaS0f+fU5iFJCyQd3U7/QySd0oVxjZR0SGfbm5lZ52wKX6S+NCKGS9oMuBs4AbipjXNfBf4deE/6KXUdcBYwB7gTOBL4DXApcEtEXCdpr3RsSBv9DwFOAW7u5FhGAmuAWZ1sbz3s2ZvHlnXeyDkTyzqvsbGxC9WYWWf1ypllayJiPfAA0NDOOX+PiD9ShOY/SRoEbBMRsyMigBuBY5ubAdukxwOAp9spYzzwgTTTPVfSZpImSvpTmpV+Nl3vK5J+nB4Pk7QoBfHngHNT+w+0qPEsSU2SmtavXV3Wc2JmZuXZFGaWAEjqBxwIfLkTzRuAFSXbK9gYuuOAmZK+BPQHRrXTz1jg/Ij4aKrpLGB1RLxPUl/gPkkzgUlAo6TjgEuAz0bEYkn/CayJiKtadhwRk4HJAH0HDY1OjNG6wY6njC/rvEb/ii6zmrYpzCx3lTQPeAF4MiIWdKIPtbKvOZBOBm6IiMHA0cBPJZX7vH4YODXVdz+wAzA0IjYAY4CfAr+LiPs6UbOZmVXIpjCzbH7PchDFbO2YiJjewT5WAINLtgez8XbrGRTvXxIRs9MMdiCwqox+BXwpIma0cmwoxfuTO3WwVjMzq7BNYWYJQEQ8Q3Eb9OJOtn1Z0kFpFeypwC/T4SeBwwEk7Qn0A55ro6uXga1LtmcAn29eoStpd0n9JQ0AvgscBuwg6fg22puZWQ/YZMIymQZs2XJxTClJy4HvAGMkrUgLawA+D1wPLAGWUqyEBTgPOFPSfGAKMCYtAmrNAuB1SfMlnZv6Www8KGkR8AOK2f7VwP+LiL9QzFzHS3o78CvguNYW+JiZWfdR23+vW73qO2hoDDptUrXLsA5Y7gU+ZlUnaW5EjGjt2KY2szQzM+uwTWGBz5tIOgKY0GL3sog4rkL9D6NYyVpqXUQcWIn+zcysZ22SYZlWn7a2ArVS/S8EhndX/2Zm1rN8G9bMzCzDYWlmZpbhsDQzM8twWJqZmWVskgt8erthDQNo8uf2zMwqxjNLMzOzDIelmZlZhsPSzMwsw2FpZmaW4bA0MzPL8GrYXmjhytUMGXtHtcswexP/dhWrV55ZmpmZZTgszczMMhyWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4bA0MzPLcFiamZllOCzNzMwyHJZmZmYZNRWWkta02B4j6dr0eJyklZLmSVos6eS0f5mkPVq0myTpwp6rvDyShks6ugvtj5W0VyVrMjOzvHr7IvWrI+IqSUOBuZJuA34OnAR8HUBSH+B44NDuKEDSZhGxvpPNhwMjgDs72f5Y4NfA4k62N+u0Z28e2+U+Rs6Z2KX2jY2NXa7BrDNqamZZroh4DFgLbAdMoQjLZocByyPiidbaptnq7ZLukvSYpG+VHDtZ0kJJiyRNKNm/RtJlku4HDk7bEyTNlXSPpAMkNUp6XNIxbVz3rcBlwIlpdnyipP6SfizpT5IekjQ6nXuNpK+lx0dI+r2kQ4BjgImp/a4t+j9LUpOkpvVrV3f4OTUzs7bV2sxyC0nzSra3B6a3PEnSfsBjEbEKWCVpg6R9ImI+RXBOyVxnOLAvsA54VNL3gPXABGB/4CVgpqRjI2Ia0B9YFBHNAdYfaIyIiyRNBa4A/hXYC/hJazVHxD9SAI6IiLNTP98E/jsiTpe0LfCApHuAscCfJP0BuAY4OiKWSpoO/Doibmul/8nAZIC+g4ZGZvxmHbbjKeO73Eejf0WX1alaC8tXImJ484akMRS3LZudK+lM4F3AkSX7pwAnSXoYGA18LXOd30bE6nSNxcAuwA4UAfhc2n8TxSx1GkWQ/qKk/T+Au9LjhcC6iHhN0kJgSJljBfgwcIyk89N2P2DniPhzGufvgXMjYmkH+jQzswqrtbDMaX7P8v8AN0raNSJepQjLmcDvgAVpxtmedSWP11M8D2rn/FdbvE/5WkQ0z942NPcXERskdeQ5FfDxiHi0lWPDgBeAnTrQn5mZdYN6fc/ydqAJOC1tL6UIlvHkb8G25X7gg5IGStoMOJkifCvpZWDrku0ZwJckCUDSvunPXYDzKG4VHyXpwDbam5lZD6jLsEwuA76SVr9CEZLvBqZ2prOIeAa4GLgXmA88GBG/rEShJe4F9mpe4ANcDmwOLJC0CLg8BeePgPMj4mngDOB6Sf0oVv5ekBYD7drGNczMrMK08W6i9RZ9Bw2NQadNqnYZZm+y3At8rIZJmhsRI1o7Vs8zSzMzsx5Rbwt8yibpCIqPgpRaFhHH9eZrm5lZ5fXasIyIGRQLaDapa5uZWeX5NqyZmVmGw9LMzCzDYWlmZpbhsDQzM8votQt8NmXDGgbQ5M+zmZlVjGeWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4dWwvdDClasZMvaOapdh1mP820ysu3lmaWZmluGwNDMzy3BYmpmZZTgszczMMhyWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4bA0MzPLcFiamZllbFJhKWlNi+0xkq5Nj8dJWilpnqTFkk5O+5dJ2qNFu0mSLuy5ys3MrJr8RepvdHVEXCVpKDBX0m3Az4GTgK8DSOoDHA8c2h0FSNosItZ3R9/Wuz1789hql1A1I+dMrHYJVdXY2FjtEnq9TWpmWa6IeAxYC2wHTKEIy2aHAcsj4onW2qbZ6u2S7pL0mKRvlRw7WdJCSYskTSjZv0bSZZLuBw5O2xMkzZV0j6QDJDVKelzSMW1c9yxJTZKa1q9dXYFnwczMmm1qM8stJM0r2d4emN7yJEn7AY9FxCpglaQNkvaJiPkUwTklc53hwL7AOuBRSd8D1gMTgP2Bl4CZko6NiGlAf2BRRHwtXb8/0BgRF0maClwB/CuwF/CT1mqOiMnAZIC+g4ZGGc+F9TI7njK+2iVUTaN/RZd1s00tLF+JiOHNG5LGACNKjp8r6UzgXcCRJfunACdJehgYDXwtc53fRsTqdI3FwC7ADhQB+FzafxPFLHUaRZD+oqT9P4C70uOFwLqIeE3SQmBImWM1M7MK8W3YN7o6IvYATgRulNQv7Z8CnACMAhakGWd71pU8Xk/xjxK1c/6rLd6nfC0immeHG5r7i4gNbHr/wDEzqzqHZSsi4nagCTgtbS8FXgDGk78F25b7gQ9KGihpM+Bk4HcVKNfMzLqZw7JtlwFfSatfoQjJdwNTO9NZRDwDXAzcC8wHHoyIX1aiUDMz617aeLfPeou+g4bGoNMmVbsMsx6z3At8rAIkzY2IEa0d88zSzMwsw4tFOknSERQfBSm1LCKOq0Y9ZmbWfRyWnRQRM4AZ1a7DzMy6n2/DmpmZZTgszczMMhyWZmZmGQ5LMzOzDC/w6YWGNQygyZ87MzOrGM8szczMMhyWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4Y+O9EILV65myNg7ql2GWc3zr/aycnlmaWZmluGwNDMzy3BYmpmZZTgszczMMhyWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4bA0MzPLcFh2gqQ16c8hkl6RNE/SYkk3Stq8nXY7SLpX0hpJ17Y4tr+khZKWSLpGktL+nVObhyQtkHR0947OzMxa8nfDdt3SiBguaTPgbuAE4KY2zn0V+HfgPemn1HXAWcAc4E7gSOA3wKXALRFxnaS90rEhlR6E9R7P3jy22iXUjZFzJla7hLrQ2NhY7RKqzjPLComI9cADQEM75/w9Iv5IEZr/JGkQsE1EzI6IAG4Ejm1uBmyTHg8Anm6tb0lnSWqS1LR+7eoujcXMzN7IM8sKkdQPOBD4cieaNwArSrZXsDF0xwEzJX0J6A+Maq2DiJgMTAboO2hodKIG6yV2PGV8tUuoG43+rSNWJs8su25XSfOAF4AnI2JBJ/pQK/uaA+9k4IaIGAwcDfxUkl83M7Me5L90u25pRAwHdgMOknRMJ/pYAQwu2R7MxtutZwC3AETEbKAfMLDT1ZqZWYc5LCskIp4BxgIXd7Lty5IOSqtgTwV+mQ4/CRwOIGlPirB8riJFm5lZWRyWlTUN2FLSB9o6QdJy4DvAGEkr0gpXgM8D1wNLgKUUK2EBzgPOlDQfmAKMSYuAzMysh3iBTydExFbpz+WUfAQkhdg+mbZD2tjfxJs/TkJELAYO7Xy1ZmbWVZ5ZmpmZZXhm2Q0kHQFMaLF7WUQcV416zMysaxyW3SAiZgAzql2HmZlVhm/DmpmZZTgszczMMhyWZmZmGQ5LMzOzDC/w6YWGNQygyV8QbWZWMZ5ZmpmZZTgszczMMhyWZmZmGQ5LMzOzDIelmZlZhsPSzMwswx8d6YUWrlzNkLF3VLsMM+uA5f64V03zzNLMzCzDYWlmZpbhsDQzM8twWJqZmWU4LM3MzDIclmZmZhkOSzMzswyHpZmZWYbD0szMLMNhaWZmllH1sJR0taR/K9meIen6ku1vSwpJe7RoN0nShe30e7GkJZIelXREyf63Spos6S+SHpH08bT/K5IWS1og6beSdmmj33GSzk+Pb5C0TNI8SfMlHZ4Z69mpppA0sGS/JF2Tji2QtF/JsXMlPSxpkaQpkvq1dw0zM6u8Wvhu2FnAJ4BJkvoAA4FtSo4fAtwFnAR8HSCddzxwaGsdStornb83sBNwj6TdI2I9cAmwKiJ2T/1sn5o9BIyIiLWSPg98CzixjPoviIjbJH0ImAwMbefc+4BfA40t9h+V2g0FDgSuAw6U1ACcA+wVEa9IuiWN64Yy6jLrUc/ePLbaJdS1kXMmVruEutfY2NhtfVd9ZkkRIIekx3sDi4CXJW0nqS+wJ3ARRUg0OwxYHhFPtNHnaODnEbEuIpYBS4AD0rHTgSsBImJDRDyfHt8bEWvTOXOAwR0cx2ygob0TIuKhiFjeRr03RmEOsK2kQenYW4AtJL0F2BJ4urW+JZ0lqUlS0/q1qztYupmZtafqM8uIeFrS65J2pgjN5tA5GFgNLIiIBZI2SNonIuZTBOeUdrptoAi8ZiuABknbpu3LJY0ElgJnR8RfW7Q/A/hNB4dyJDCtg22aNQBPlWyvABoioknSVcCTwCvAzIiY2VoHETGZYmZL30FDo5N1mHXajqeMr3YJda3Rv3WkptXCzBI2zi6bw3J2yfasdM4U4KQ0wxoN3NpOf2plX1D842AwcF9E7Jeuc9UbGkqfBEYA5d4TmSjpceBnwDfLbFNWvZK2oxjrOyluJ/dP9ZmZWQ+qlbCcRRGMwyhuw86hmFkeQhGkUITlCcAoitnmqnb6WwG8o2R7MMXtyxeAtcDUtP9WoHQxzSiK9zSPiYh1ad830gKeeW1c6wJgN+BS4CdljLUj9Y4ClkXEcxHxGnA7G29Zm5lZD6mVsLwP+CjwYkSsj4gXgW0pAnM2QEQspQi78bR/CxZgOsUstK+kd1IsnHkgIgL4FTAynXc4sBhA0r7ADyiC8p9BHBGXRMTwiBje1sUiYgPwXaBP6crbDpgOnJpWxR4ErI6IZyhuvx4kaUtJSvX+uRP9m5lZF9RKWC6kWAU7p8W+1c0LcJIpwLvZODNsVUQ8DNxCEYR3AV9MK2GhWCw0TtIC4FPAeWn/RGAr4NY0k5zekQGkIL4CaO/jLOdIWkExc1xQ8hGZO4HHKRYi/RD4QurzfuA24EGK56MP6X1JMzPrOSr+jrfepO+goTHotEnVLsPMOmC5F/hUnaS5ETGitWO1MrM0MzOrWVX/6EhXpPcHJ7TYvSwijqtGPc0kTaVYwVrqooiYUY16zMysa+o6LFP41FwAVTuszcyssnwb1szMLMNhaWZmluGwNDMzy3BYmpmZZdT1Ah9r3bCGATT5M1tmZhXjmaWZmVmGw9LMzCzDYWlmZpbhsDQzM8twWJqZmWU4LM3MzDIclmZmZhkOSzMzswyHpZmZWYYioto1WIVJehl4tNp1VMBA4PlqF9FFHkNt6A1jgN4xjloewy4R8bbWDvjr7nqnRyNiRLWL6CpJTfU+Do+hNvSGMUDvGEe9jsG3Yc3MzDIclmZmZhkOy95pcrULqJDeMA6PoTb0hjFA7xhHXY7BC3zMzMwyPLM0MzPLcFiamZllOCzrjKQjJT0qaYmksa0cl6Rr0vEFkvYrt21P6eIYlktaKGmepKaerfwNNebG8G5JsyWtk3R+R9r2pC6Oo15ei/+b/jtaIGmWpH3KbdtTujiGenkdRqf650lqkvT+ctvWhIjwT538AJsBS4F3AW8F5gN7tTjnaOA3gICDgPvLbVvrY0jHlgMD6+B1eDvwPuAbwPkdaVsP46iz1+IQYLv0+Kg6/X+i1THU2euwFRvXybwXeKSWXofcj2eW9eUAYElEPB4R/wB+Doxucc5o4MYozAG2lTSozLY9oStjqBXZMUTEqoj4E/BaR9v2oK6Mo1aUM4ZZEfFS2pwDDC63bQ/pyhhqRTljWBMpHYH+QJTbthY4LOtLA/BUyfaKtK+cc8pp2xO6MgYo/gebKWmupLO6rcr2deW5rJXXoRK11ONrcQbFXYvOtO0uXRkD1NHrIOk4SY8AdwCnd6Rttfnr7uqLWtnX8rM/bZ1TTtue0JUxABwaEU9Lejtwt6RHIuL3Fa0wryvPZa28DtD1WurqtZD0IYqgaX6vrFZei66MAerodYiIqcBUSYcBlwOjym1bbZ5Z1pcVwDtKtgcDT5d5Tjlte0JXxkBENP+5CphKcQunp3XluayV16HLtdTTayHpvcD1wOiIeKEjbXtAV8ZQV69DsxTmu0oa2NG2VVPtN039U/4PxZ2Ax4F3svGN8L1bnPMR3rg45oFy29bBGPoDW5c8ngUcWYtjKDl3HG9c4FMTr0MFxlE3rwWwM7AEOKSz46/hMdTT67AbGxf47AesTP+P18TrkB1jtQvwTwdfsGKl6F8oVo9dkvZ9Dvhceizg++n4QmBEe23raQwUq+Xmp5+Ha3wMO1L8i/l/gL+lx9vU0uvQlXHU2WtxPfASMC/9NLXXtp7GUGevw0WpxnnAbOD9tfY6tPfjr7szMzPL8HuWZmZmGQ5LMzOzDIelmZlZhsPSzMwsw2FpZmaW4bA0MzPLcFiamZll/C8n3T8lPfclJgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.barh(np.arange(6), f_importance_ordered, xerr=std[inds], align=\"center\")\n",
    "plt.yticks(np.arange(6), features_ordered)\n",
    "plt.title('Feature importance')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}