Appendix02_Random_Forest-Train_Val_Split.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4dde78e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "from keras.datasets import mnist\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "tf.config.run_functions_eagerly(True)\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a085f070",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000,)\n",
      "(50000, 1568)\n",
      "(10000, 1568)\n"
     ]
    }
   ],
   "source": [
    "train_X = pd.read_csv('train.csv')\n",
    "test_X = pd.read_csv('test.csv' )\n",
    "train_X.drop(train_X.columns[train_X.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
    "test_X.drop(test_X.columns[test_X.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
    "\n",
    "train_result = pd.read_csv('train_result.csv')\n",
    "train_y_class = train_result['Class']\n",
    "\n",
    "X_train = train_X.to_numpy()\n",
    "X_test = test_X.to_numpy()\n",
    "train_y = train_y_class.to_numpy()\n",
    "\n",
    "print(train_y.shape)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "00cef114",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_val, train_y, val_y = train_test_split(X_train, train_y,  test_size=0.20, random_state=101)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "eec36728",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40000, 1568)\n",
      "(10000, 1568)\n",
      "(40000,)\n",
      "(10000,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_val.shape)\n",
    "print(train_y.shape)\n",
    "print(val_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e550f1c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(n_estimators=250)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf_model = RandomForestClassifier(n_estimators=250)\n",
    "\n",
    "rf_model.fit(X_train, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c1d5db58",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e3a0ddbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ab59c949",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 92   0   0   0   0   1   5   0   4   0   0   0   0   0   0   0   0   0\n",
      "    0]\n",
      " [  0 186   3   2   1   1   0   4   0   3   2   0   0   0   0   0   0   0\n",
      "    0]\n",
      " [  3   4 295   4   1   1   8   8  13   2   3   1   0   0   0   0   0   0\n",
      "    0]\n",
      " [  1   7   2 360   3   9   5  12  18  15   1   6   0   0   0   0   0   0\n",
      "    0]\n",
      " [  1   1   4  13 423   3  21   5  16  33  10   4   3   0   0   0   0   0\n",
      "    0]\n",
      " [  1   5   2  12  17 445   7  19  18  19  26  25   1   0   0   0   0   0\n",
      "    0]\n",
      " [  1   1   3   1  18  18 536  12  21  13  22  31   9   2   3   0   1   0\n",
      "    0]\n",
      " [  0   5  10   4   4  26  25 608  24  26  14  24  39   6   0   2   0   0\n",
      "    0]\n",
      " [  0   7   4   9  15   8  26  15 735  16  21  14   6  34   7   0   0   0\n",
      "    0]\n",
      " [  2   1   4   8  13  11  12  30  31 771  27  24  20  12  12   7   2   0\n",
      "    0]\n",
      " [  0   2   3  17  16  23   8  10  38  30 693  23  37   6   7  15   9   1\n",
      "    0]\n",
      " [  0   0   0   1   5  11  28  16  20  57  30 541  11  20   3   7   7   5\n",
      "    0]\n",
      " [  0   0   0   1   1  11  13  24  10  18  47  25 523  13   9   2   5   4\n",
      "    0]\n",
      " [  0   0   0   0   0   3  15   7  30  19  12  47  16 407   5  13   0   1\n",
      "    3]\n",
      " [  0   0   0   0   0   0   0  20  23  40  16   8  50   9 280   2  10   0\n",
      "    1]\n",
      " [  0   0   0   0   0   0   1   6  11  28  17   9  21  24   2 241   6   1\n",
      "    0]\n",
      " [  0   0   0   0   0   0   0   1   6  17  17  24   8  10   5   1 201   1\n",
      "    1]\n",
      " [  0   0   0   0   0   0   0   0   3   7  11   8  48   5   2  11   1  90\n",
      "    4]\n",
      " [  0   0   0   0   0   0   0   1   1   3   0   1   1  19   1   0  12   0\n",
      "   36]]\n",
      "sum_preds               : 10000\n",
      "sum_correct_predictions : 7463\n",
      "sum_wrong_predictions   : 2537\n",
      "misclassification_error : 0.25370000000000004\n",
      "Accuracy\n",
      "0.7463\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.90      0.91       102\n",
      "           1       0.85      0.92      0.88       202\n",
      "           2       0.89      0.86      0.88       343\n",
      "           3       0.83      0.82      0.83       439\n",
      "           4       0.82      0.79      0.80       537\n",
      "           5       0.78      0.75      0.76       597\n",
      "           6       0.75      0.77      0.76       692\n",
      "           7       0.76      0.74      0.75       817\n",
      "           8       0.72      0.80      0.76       917\n",
      "           9       0.69      0.78      0.73       987\n",
      "          10       0.72      0.74      0.73       938\n",
      "          11       0.66      0.71      0.69       762\n",
      "          12       0.66      0.74      0.70       706\n",
      "          13       0.72      0.70      0.71       578\n",
      "          14       0.83      0.61      0.70       459\n",
      "          15       0.80      0.66      0.72       367\n",
      "          16       0.79      0.69      0.74       292\n",
      "          17       0.87      0.47      0.61       190\n",
      "          18       0.80      0.48      0.60        75\n",
      "\n",
      "    accuracy                           0.75     10000\n",
      "   macro avg       0.78      0.73      0.75     10000\n",
      "weighted avg       0.75      0.75      0.75     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = rf_model.predict(X_val)\n",
    "\n",
    "rf_matrix = confusion_matrix(val_y,predictions)\n",
    "print(rf_matrix)\n",
    "\n",
    "#misclassification_error(confusion_matrix):\n",
    "sum_preds = np.sum(rf_matrix)\n",
    "sum_correct = np.sum(np.diag(rf_matrix))\n",
    "misclassification_error = 1.0 - (float(sum_correct) / float(sum_preds))\n",
    "    \n",
    "print(\"sum_preds               :\", int(sum_preds)) \n",
    "print(\"sum_correct_predictions :\", int(sum_correct)) \n",
    "print(\"sum_wrong_predictions   :\", int(sum_preds - sum_correct))\n",
    "print(\"misclassification_error :\", misclassification_error)\n",
    "\n",
    "# measure accuracy\n",
    "print(\"Accuracy\")\n",
    "print(accuracy_score(val_y,predictions))\n",
    "\n",
    "# class-wise accuracy\n",
    "class_report = classification_report(val_y,predictions)\n",
    "print(class_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3b7bd749",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bootstrap': [True, False],\n",
      " 'criterion': ['gini', 'entropy'],\n",
      " 'max_depth': [10, 30, 60, 90, None],\n",
      " 'max_features': ['auto', 'sqrt'],\n",
      " 'min_samples_leaf': [1, 2, 4],\n",
      " 'min_samples_split': [2, 5, 10],\n",
      " 'n_estimators': [100, 200, 300]}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "# Number of trees in random forest\n",
    "#n_estimators = [int(x) for x in np.linspace(start = 10, stop = 2000, num = 20)]\n",
    "n_estimators = [100, 200, 300]\n",
    "# Number of features to consider at every split\n",
    "max_features = ['auto', 'sqrt']\n",
    "\n",
    "# Maximum number of levels in tree\n",
    "#max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n",
    "max_depth = [10, 30, 60, 90]\n",
    "max_depth.append(None)\n",
    "\n",
    "# Minimum number of samples required to split a node\n",
    "min_samples_split = [2, 5, 10]\n",
    "\n",
    "# Minimum number of samples required at each leaf node\n",
    "min_samples_leaf = [1, 2, 4]\n",
    "\n",
    "# Method of selecting samples for training each tree\n",
    "bootstrap = [True, False]\n",
    "\n",
    "# Criterion\n",
    "criterion=['gini', 'entropy']\n",
    "\n",
    "# Create the random grid\n",
    "random_grid = {'n_estimators': n_estimators,\n",
    "               'max_features': max_features,\n",
    "               'max_depth': max_depth,\n",
    "               'min_samples_split': min_samples_split,\n",
    "               'min_samples_leaf': min_samples_leaf,\n",
    "               'bootstrap': bootstrap,\n",
    "               'criterion': criterion}\n",
    "\n",
    "pprint(random_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "b2f5e0f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 100 candidates, totalling 300 fits\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=3, estimator=RandomForestClassifier(), n_iter=100,\n",
       "                   n_jobs=-1,\n",
       "                   param_distributions={'bootstrap': [True, False],\n",
       "                                        'criterion': ['gini', 'entropy'],\n",
       "                                        'max_depth': [10, 30, 60, 90, None],\n",
       "                                        'max_features': ['auto', 'sqrt'],\n",
       "                                        'min_samples_leaf': [1, 2, 4],\n",
       "                                        'min_samples_split': [2, 5, 10],\n",
       "                                        'n_estimators': [100, 200, 300]},\n",
       "                   random_state=42, verbose=2)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the random grid to search for best hyperparameters\n",
    "# First create the base model to tune\n",
    "\n",
    "rf_model = RandomForestClassifier()\n",
    "\n",
    "# Random search of parameters, using 3 fold cross validation, \n",
    "# search across 100 different combinations, and use all available cores\n",
    "rf_random = RandomizedSearchCV(estimator = rf_model, param_distributions = random_grid, n_iter = 100, cv = 3, verbose=2, random_state=42, n_jobs = -1)\n",
    "# Fit the random search model\n",
    "rf_random.fit(X_train, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e1b4e773",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 300,\n",
       " 'min_samples_split': 2,\n",
       " 'min_samples_leaf': 1,\n",
       " 'max_features': 'sqrt',\n",
       " 'max_depth': 90,\n",
       " 'criterion': 'gini',\n",
       " 'bootstrap': False}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_random.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a856d7c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_best_param_model = RandomForestClassifier(**rf_random.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "98083ef4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=False, max_depth=90, max_features='sqrt',\n",
       "                       n_estimators=300)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_best_param_model.fit(X_train, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "98266d9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = rf_best_param_model.predict(X_val)\n",
    "\n",
    "rf_matrix = confusion_matrix(val_y,predictions)\n",
    "\n",
    "\n",
    "#misclassification_error(confusion_matrix):\n",
    "sum_preds = np.sum(rf_matrix)\n",
    "sum_correct = np.sum(np.diag(rf_matrix))\n",
    "misclassification_error = 1.0 - (float(sum_correct) / float(sum_preds))\n",
    "val_accuracy = accuracy_score(val_y,predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "65d976e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7813\n"
     ]
    }
   ],
   "source": [
    "print (val_accuracy )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fa497b0",
   "metadata": {},
   "source": [
    "# Retrain using whole train dataset and predictions on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4a0993dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50000,)\n",
      "(50000, 1568)\n",
      "(10000, 1568)\n"
     ]
    }
   ],
   "source": [
    "train_X = pd.read_csv('train.csv')\n",
    "test_X = pd.read_csv('test.csv' )\n",
    "train_X.drop(train_X.columns[train_X.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
    "test_X.drop(test_X.columns[test_X.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)\n",
    "\n",
    "train_result = pd.read_csv('train_result.csv')\n",
    "train_y_class = train_result['Class']\n",
    "\n",
    "X_train = train_X.to_numpy()\n",
    "X_test = test_X.to_numpy()\n",
    "train_y = train_y_class.to_numpy()\n",
    "\n",
    "print(train_y.shape)\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9f6c5470",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=False, max_depth=90, max_features='sqrt',\n",
       "                       n_estimators=300)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_model_final = RandomForestClassifier(n_estimators=300, max_depth=90, min_samples_split=2,\n",
    "                                     min_samples_leaf=1, bootstrap=False, criterion='gini', max_features='sqrt' )\n",
    "\n",
    "rf_model_final.fit(X_train, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "36c6911a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Test Predictions\n",
      "[10  7 10 ... 10  4  6]\n",
      "10000\n",
      "<class 'numpy.ndarray'>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Index</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9995</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9996</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9997</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9998</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9999</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Class\n",
       "Index       \n",
       "0         10\n",
       "1          7\n",
       "2         10\n",
       "3          7\n",
       "4          5\n",
       "...      ...\n",
       "9995      12\n",
       "9996      12\n",
       "9997      10\n",
       "9998       4\n",
       "9999       6\n",
       "\n",
       "[10000 rows x 1 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('\\n')\n",
    "print(\"Test Predictions\")\n",
    "test_predictions = rf_model.predict(X_test)\n",
    "print(test_predictions)\n",
    "print(len(test_predictions))\n",
    "print(type(test_predictions))\n",
    "df = pd.DataFrame(test_predictions)\n",
    "df.index.name= 'Index'\n",
    "df.columns = ['Class']\n",
    "file_name = f'test_predictions_random_forest_n300_depth90.csv'\n",
    "df.to_csv(file_name, index=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a134abb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}