Lesson7SVM.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lesson7SVM.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "x3mKX2_iEKka",
        "2TRLdTcZRn1_"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hbechara/HertieML/blob/main/Lesson7SVM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x3mKX2_iEKka"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G1uzJHk_EKka"
      },
      "source": [
        "First, let's import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures. We also check that Python 3.5 or later is installed (although Python 2.x may work, it is deprecated so we strongly recommend you use Python 3 instead), as well as Scikit-Learn ≥0.20."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qn3x_afZEKka"
      },
      "source": [
        "# Python ≥3.5 is required\n",
        "import sys\n",
        "assert sys.version_info >= (3, 5)\n",
        "\n",
        "# Scikit-Learn ≥0.20 is required\n",
        "import sklearn\n",
        "assert sklearn.__version__ >= \"0.20\"\n",
        "\n",
        "# Common imports\n",
        "import numpy as np\n",
        "import os\n",
        "\n",
        "# to make this notebook's output stable across runs\n",
        "np.random.seed(42)\n",
        "\n",
        "# To plot pretty figures\n",
        "%matplotlib inline\n",
        "import matplotlib as mpl\n",
        "import matplotlib.pyplot as plt\n",
        "mpl.rc('axes', labelsize=14)\n",
        "mpl.rc('xtick', labelsize=12)\n",
        "mpl.rc('ytick', labelsize=12)\n",
        "\n",
        "# Where to save the figures\n",
        "PROJECT_ROOT_DIR = \".\"\n",
        "CHAPTER_ID = \"svm\"\n",
        "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
        "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
        "\n",
        "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
        "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
        "    print(\"Saving figure\", fig_id)\n",
        "    if tight_layout:\n",
        "        plt.tight_layout()\n",
        "    plt.savefig(path, format=fig_extension, dpi=resolution)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2TRLdTcZRn1_"
      },
      "source": [
        "# Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPZjKigWRv2O"
      },
      "source": [
        "from sklearn.datasets import fetch_california_housing\n",
        "\n",
        "housing = fetch_california_housing()\n",
        "X = housing[\"data\"]\n",
        "y = housing[\"target\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pKgHlvZoR4vb"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qmkqmKrLEKku"
      },
      "source": [
        "# Preprocessing: \n",
        "\n",
        "Don't forget to scale the data:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZlXjqMvrSE16"
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "scaler = StandardScaler()\n",
        "X_train_scaled = scaler.fit_transform(X_train)\n",
        "X_test_scaled = scaler.transform(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qpGQ4ZZDRztS"
      },
      "source": [
        "# Baseline: Train a Linear SVR as a baseline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "45oJ5_5FSPCQ",
        "outputId": "76023232-cb50-48af-b738-ab7d728b488b"
      },
      "source": [
        "from sklearn.svm import LinearSVR\n",
        "\n",
        "lin_svr = LinearSVR(random_state=42)\n",
        "lin_svr.fit(X_train_scaled, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/svm/_base.py:947: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
            "  \"the number of iterations.\", ConvergenceWarning)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "LinearSVR(C=1.0, dual=True, epsilon=0.0, fit_intercept=True,\n",
              "          intercept_scaling=1.0, loss='epsilon_insensitive', max_iter=1000,\n",
              "          random_state=42, tol=0.0001, verbose=0)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yZfUp9W3ScKT"
      },
      "source": [
        "# Step 2: use MSE to test how it performs on the train set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_5j1xByvSheH",
        "outputId": "40669746-61a8-4b65-c024-65b590ec0983"
      },
      "source": [
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "y_pred = lin_svr.predict(X_train_scaled)\n",
        "mse = mean_squared_error(y_train, y_pred)\n",
        "mse"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.949968822217229"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tp-bvFwocBId"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4MwK9laXS4IO"
      },
      "source": [
        "# RBF Kernel\n",
        "\n",
        "Now try a regressor with the RBF Kernel. \n",
        "\n",
        "Use randomized search with cross validation to find the appropriate hyperparameter values for C and gamma.\n",
        "\n",
        "I've imported everything you need, so you know where to look for hints!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Mkld8_WS4bM"
      },
      "source": [
        "from sklearn.svm import SVR\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "from scipy.stats import reciprocal, uniform\n",
        "\n",
        "\n",
        "rbf_svr = SVR(C=5.0, cache_size=200, coef0=0.0, degree=3,\n",
        "                                 epsilon=0.1, gamma=0.002, kernel='rbf',\n",
        "                                 max_iter=-1, shrinking=True, tol=0.001,\n",
        "                                 verbose=False)\n",
        "rbf_svr.fit(X_train_scaled, y_train)\n",
        "\n",
        "y_pred = rbf_svr.predict(X_test_scaled)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JMXRq59QcCb5",
        "outputId": "634c9c63-5cd5-4074-d920-a2bcb97a38ec"
      },
      "source": [
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "mse = mean_squared_error(y_test, y_pred)\n",
        "mse"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.48718861287031906"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GgYVcTxfjWE1",
        "outputId": "8a3187f2-e440-4070-afc4-5500af373f6b"
      },
      "source": [
        "\n",
        "np.sqrt(mse)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6979889776137722"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iF-wx92GcsmO"
      },
      "source": [
        "from sklearn.svm import SVR\n",
        "from sklearn.model_selection import RandomizedSearchCV\n",
        "from scipy.stats import reciprocal, uniform\n",
        "\n",
        "param_distributions = {\"gamma\": reciprocal(0.001, 0.1), \"C\": uniform(1, 10)}\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L0eHPuigfcKi"
      },
      "source": [
        "Run the search and train."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "id": "m4SWHT11edVm",
        "outputId": "653d4639-fe2d-4b60-94ff-8ee55b2896f9"
      },
      "source": [
        "rnd_search_cv = RandomizedSearchCV(SVR(), param_distributions, n_iter=10, verbose=2, cv=3, random_state=42)\n",
        "rnd_search_cv.fit(X_train_scaled, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-540d8b544513>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrnd_search_cv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRandomizedSearchCV\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSVR\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_distributions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_iter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mrnd_search_cv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_scaled\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'RandomizedSearchCV' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JE4Jvv-sTXCR"
      },
      "source": [
        "# Finally: Test this model and compare to the baseline model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "kGAXhgTqTnL4",
        "outputId": "074090b4-2cbe-442f-929d-818e1d7df7aa"
      },
      "source": [
        "y_pred = rnd_search_cv.best_estimator_.predict(X_train_scaled)\n",
        "mse = mean_squared_error(y_train, y_pred)\n",
        "np.sqrt(mse)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-4a9a01d93638>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnd_search_cv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbest_estimator_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_scaled\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mmse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmean_squared_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'rnd_search_cv' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sy-7tcJlU2kV",
        "outputId": "02696d09-d68e-41ef-ff22-56e0f56326e1"
      },
      "source": [
        "y_pred = rnd_search_cv.best_estimator_.predict(X_test_scaled)\n",
        "mse = mean_squared_error(y_test, y_pred)\n",
        "np.sqrt(mse)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.592916838552874"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    }
  ]
}