Lesson8Lab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lesson8Lab.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hbechara/HertieML/blob/main/Lesson8Lab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5UCFZDTi8-dm"
      },
      "source": [
        "_Exercise: train and fine-tune a Decision Tree for the moons dataset._"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hFNMpmZl-7ti"
      },
      "source": [
        "a. Generate a moons dataset using `make_moons(n_samples=10000, noise=0.4)`."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Dl_HLdYP-_cb"
      },
      "source": [
        "Take a look at make_moons in sklearn documentation to find out what you are doing in this cell:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gp5pE7R78-dm"
      },
      "source": [
        "from sklearn.datasets import make_moons\n",
        "\n",
        "X, y = make_moons(n_samples=10000, noise=0.4, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6qfYFuPB_b7Q"
      },
      "source": [
        "b. Split it into a training set and a test set by either\n",
        "  - using `train_test_split()`.\n",
        "  - or writing your own method."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2BsRXy_B_iXc"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 ,random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hVDrB2x7_MYb"
      },
      "source": [
        "c. Use grid search with cross-validation (with the help of the `GridSearchCV` class) to find good hyperparameter values for a `DecisionTreeClassifier`. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00WRg6Bg_1kC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "907b6de5-88ce-4572-eb1b-87f1a431f54a"
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "\n",
        "params = {'max_leaf_nodes': list(range(2, 100)), 'min_samples_split': [2, 3, 4]}\n",
        "grid_search_cv = GridSearchCV(DecisionTreeClassifier(random_state=42), params, verbose=1, cv=3)\n",
        "\n",
        "grid_search_cv.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fitting 3 folds for each of 294 candidates, totalling 882 fits\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n",
            "[Parallel(n_jobs=1)]: Done 882 out of 882 | elapsed:    9.6s finished\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GridSearchCV(cv=3, error_score=nan,\n",
              "             estimator=DecisionTreeClassifier(ccp_alpha=0.0, class_weight=None,\n",
              "                                              criterion='gini', max_depth=None,\n",
              "                                              max_features=None,\n",
              "                                              max_leaf_nodes=None,\n",
              "                                              min_impurity_decrease=0.0,\n",
              "                                              min_impurity_split=None,\n",
              "                                              min_samples_leaf=1,\n",
              "                                              min_samples_split=2,\n",
              "                                              min_weight_fraction_leaf=0.0,\n",
              "                                              presort='deprecated',\n",
              "                                              random_state=42,\n",
              "                                              splitter='best'),\n",
              "             iid='deprecated', n_jobs=None,\n",
              "             param_grid={'max_leaf_nodes': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,\n",
              "                                            13, 14, 15, 16, 17, 18, 19, 20, 21,\n",
              "                                            22, 23, 24, 25, 26, 27, 28, 29, 30,\n",
              "                                            31, ...],\n",
              "                         'min_samples_split': [2, 3, 4]},\n",
              "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
              "             scoring=None, verbose=1)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNbHoO4ykEr2",
        "outputId": "81576360-9461-4e7c-8379-1c635d3ae9eb"
      },
      "source": [
        "print(grid_search_cv.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'max_leaf_nodes': 17, 'min_samples_split': 2}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "85_ZZJx1i_Lz"
      },
      "source": [
        "# Thilos solution\n",
        "\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "\n",
        "tree_clf = DecisionTreeClassifier()\n",
        "\n",
        "param_grid = [\n",
        "   {\"max_depth\": [ 1, 2, 3], \"min_samples_split\": [ 2, 6, 10], \"min_samples_leaf\": [1, 3, 5], \"criterion\": [\"gini\", \"entropy\"]}           \n",
        "]\n",
        "\n",
        "grid_search = GridSearchCV(tree_clf, param_grid, cv=3, return_train_score = True, verbose=2)\n",
        "grid_search.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q6diTOR-ACRt"
      },
      "source": [
        "d. Train it on the full training set using these hyperparameters, and measure your model's performance on the test set. \n",
        "\n",
        "How well did your classifier do? Find out by calculating the accuracy on the test set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFTke06IVmxc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d560a15a-ac60-4936-beba-9c0d29ec6ad3"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "y_pred = grid_search_cv.predict(X_test)\n",
        "accuracy_score(y_test, y_pred)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8695"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hHfi2AbBxy9a"
      },
      "source": [
        "# Bruno's solution\n",
        "\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.model_selection import GridSearchCV, cross_val_score\n",
        "\n",
        "DTM = DecisionTreeClassifier()\n",
        "\n",
        "param_grid = {'criterion':['gini','entropy'], 'max_depth':[2,4,8,16,32],'max_leaf_nodes': list(range(2, 100))}\n",
        "\n",
        "grid_search = GridSearchCV(DTM, param_grid=param_grid, cv=5)\n",
        "grid_search.fit(X_train, y_train)\n",
        "\n",
        "grid_search.cv_results_['params']\n",
        "\n",
        "print(grid_search.best_params_)\n",
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "DTM = DecisionTreeClassifier(criterion = 'gini', max_depth = 8, max_leaf_nodes = 23)\n",
        "\n",
        "DTM.fit(X_train, y_train)\n",
        "\n",
        "y_pred = DTM.predict(X_test)\n",
        "\n",
        "accuracy_score(y_pred, y_test)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}