Feature_Selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Feature Selection.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPVRrr69r8C9HhO+ZHs0OcC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Pramodgopinathan/DATA_MINING-data_reduction/blob/main/Feature_Selection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oq5g_lua8PNp",
        "outputId": "679ad9a9-49a8-4587-db94-5567755cfa65"
      },
      "source": [
        "from sklearn.datasets import load_boston\n",
        "boston = load_boston()\n",
        "print(boston.data.shape)         # for dataset dimension\n",
        "print(boston.feature_names)      # for feature names\n",
        "print(boston.target)             # for target variable\n",
        "print(boston.DESCR)              # for data description"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(506, 13)\n",
            "['CRIM' 'ZN' 'INDUS' 'CHAS' 'NOX' 'RM' 'AGE' 'DIS' 'RAD' 'TAX' 'PTRATIO'\n",
            " 'B' 'LSTAT']\n",
            "[24.  21.6 34.7 33.4 36.2 28.7 22.9 27.1 16.5 18.9 15.  18.9 21.7 20.4\n",
            " 18.2 19.9 23.1 17.5 20.2 18.2 13.6 19.6 15.2 14.5 15.6 13.9 16.6 14.8\n",
            " 18.4 21.  12.7 14.5 13.2 13.1 13.5 18.9 20.  21.  24.7 30.8 34.9 26.6\n",
            " 25.3 24.7 21.2 19.3 20.  16.6 14.4 19.4 19.7 20.5 25.  23.4 18.9 35.4\n",
            " 24.7 31.6 23.3 19.6 18.7 16.  22.2 25.  33.  23.5 19.4 22.  17.4 20.9\n",
            " 24.2 21.7 22.8 23.4 24.1 21.4 20.  20.8 21.2 20.3 28.  23.9 24.8 22.9\n",
            " 23.9 26.6 22.5 22.2 23.6 28.7 22.6 22.  22.9 25.  20.6 28.4 21.4 38.7\n",
            " 43.8 33.2 27.5 26.5 18.6 19.3 20.1 19.5 19.5 20.4 19.8 19.4 21.7 22.8\n",
            " 18.8 18.7 18.5 18.3 21.2 19.2 20.4 19.3 22.  20.3 20.5 17.3 18.8 21.4\n",
            " 15.7 16.2 18.  14.3 19.2 19.6 23.  18.4 15.6 18.1 17.4 17.1 13.3 17.8\n",
            " 14.  14.4 13.4 15.6 11.8 13.8 15.6 14.6 17.8 15.4 21.5 19.6 15.3 19.4\n",
            " 17.  15.6 13.1 41.3 24.3 23.3 27.  50.  50.  50.  22.7 25.  50.  23.8\n",
            " 23.8 22.3 17.4 19.1 23.1 23.6 22.6 29.4 23.2 24.6 29.9 37.2 39.8 36.2\n",
            " 37.9 32.5 26.4 29.6 50.  32.  29.8 34.9 37.  30.5 36.4 31.1 29.1 50.\n",
            " 33.3 30.3 34.6 34.9 32.9 24.1 42.3 48.5 50.  22.6 24.4 22.5 24.4 20.\n",
            " 21.7 19.3 22.4 28.1 23.7 25.  23.3 28.7 21.5 23.  26.7 21.7 27.5 30.1\n",
            " 44.8 50.  37.6 31.6 46.7 31.5 24.3 31.7 41.7 48.3 29.  24.  25.1 31.5\n",
            " 23.7 23.3 22.  20.1 22.2 23.7 17.6 18.5 24.3 20.5 24.5 26.2 24.4 24.8\n",
            " 29.6 42.8 21.9 20.9 44.  50.  36.  30.1 33.8 43.1 48.8 31.  36.5 22.8\n",
            " 30.7 50.  43.5 20.7 21.1 25.2 24.4 35.2 32.4 32.  33.2 33.1 29.1 35.1\n",
            " 45.4 35.4 46.  50.  32.2 22.  20.1 23.2 22.3 24.8 28.5 37.3 27.9 23.9\n",
            " 21.7 28.6 27.1 20.3 22.5 29.  24.8 22.  26.4 33.1 36.1 28.4 33.4 28.2\n",
            " 22.8 20.3 16.1 22.1 19.4 21.6 23.8 16.2 17.8 19.8 23.1 21.  23.8 23.1\n",
            " 20.4 18.5 25.  24.6 23.  22.2 19.3 22.6 19.8 17.1 19.4 22.2 20.7 21.1\n",
            " 19.5 18.5 20.6 19.  18.7 32.7 16.5 23.9 31.2 17.5 17.2 23.1 24.5 26.6\n",
            " 22.9 24.1 18.6 30.1 18.2 20.6 17.8 21.7 22.7 22.6 25.  19.9 20.8 16.8\n",
            " 21.9 27.5 21.9 23.1 50.  50.  50.  50.  50.  13.8 13.8 15.  13.9 13.3\n",
            " 13.1 10.2 10.4 10.9 11.3 12.3  8.8  7.2 10.5  7.4 10.2 11.5 15.1 23.2\n",
            "  9.7 13.8 12.7 13.1 12.5  8.5  5.   6.3  5.6  7.2 12.1  8.3  8.5  5.\n",
            " 11.9 27.9 17.2 27.5 15.  17.2 17.9 16.3  7.   7.2  7.5 10.4  8.8  8.4\n",
            " 16.7 14.2 20.8 13.4 11.7  8.3 10.2 10.9 11.   9.5 14.5 14.1 16.1 14.3\n",
            " 11.7 13.4  9.6  8.7  8.4 12.8 10.5 17.1 18.4 15.4 10.8 11.8 14.9 12.6\n",
            " 14.1 13.  13.4 15.2 16.1 17.8 14.9 14.1 12.7 13.5 14.9 20.  16.4 17.7\n",
            " 19.5 20.2 21.4 19.9 19.  19.1 19.1 20.1 19.9 19.6 23.2 29.8 13.8 13.3\n",
            " 16.7 12.  14.6 21.4 23.  23.7 25.  21.8 20.6 21.2 19.1 20.6 15.2  7.\n",
            "  8.1 13.6 20.1 21.8 24.5 23.1 19.7 18.3 21.2 17.5 16.8 22.4 20.6 23.9\n",
            " 22.  11.9]\n",
            ".. _boston_dataset:\n",
            "\n",
            "Boston house prices dataset\n",
            "---------------------------\n",
            "\n",
            "**Data Set Characteristics:**  \n",
            "\n",
            "    :Number of Instances: 506 \n",
            "\n",
            "    :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
            "\n",
            "    :Attribute Information (in order):\n",
            "        - CRIM     per capita crime rate by town\n",
            "        - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.\n",
            "        - INDUS    proportion of non-retail business acres per town\n",
            "        - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
            "        - NOX      nitric oxides concentration (parts per 10 million)\n",
            "        - RM       average number of rooms per dwelling\n",
            "        - AGE      proportion of owner-occupied units built prior to 1940\n",
            "        - DIS      weighted distances to five Boston employment centres\n",
            "        - RAD      index of accessibility to radial highways\n",
            "        - TAX      full-value property-tax rate per $10,000\n",
            "        - PTRATIO  pupil-teacher ratio by town\n",
            "        - B        1000(Bk - 0.63)^2 where Bk is the proportion of black people by town\n",
            "        - LSTAT    % lower status of the population\n",
            "        - MEDV     Median value of owner-occupied homes in $1000's\n",
            "\n",
            "    :Missing Attribute Values: None\n",
            "\n",
            "    :Creator: Harrison, D. and Rubinfeld, D.L.\n",
            "\n",
            "This is a copy of UCI ML housing dataset.\n",
            "https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
            "\n",
            "\n",
            "This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
            "\n",
            "The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
            "prices and the demand for clean air', J. Environ. Economics & Management,\n",
            "vol.5, 81-102, 1978.   Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
            "...', Wiley, 1980.   N.B. Various transformations are used in the table on\n",
            "pages 244-261 of the latter.\n",
            "\n",
            "The Boston house-price data has been used in many machine learning papers that address regression\n",
            "problems.   \n",
            "     \n",
            ".. topic:: References\n",
            "\n",
            "   - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
            "   - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:87: FutureWarning: Function load_boston is deprecated; `load_boston` is deprecated in 1.0 and will be removed in 1.2.\n",
            "\n",
            "    The Boston housing prices dataset has an ethical problem. You can refer to\n",
            "    the documentation of this function for further details.\n",
            "\n",
            "    The scikit-learn maintainers therefore strongly discourage the use of this\n",
            "    dataset unless the purpose of the code is to study and educate about\n",
            "    ethical issues in data science and machine learning.\n",
            "\n",
            "    In this special case, you can fetch the dataset from the original\n",
            "    source::\n",
            "\n",
            "        import pandas as pd\n",
            "        import numpy as np\n",
            "\n",
            "\n",
            "        data_url = \"http://lib.stat.cmu.edu/datasets/boston\"\n",
            "        raw_df = pd.read_csv(data_url, sep=\"\\s+\", skiprows=22, header=None)\n",
            "        data = np.hstack([raw_df.values[::2, :], raw_df.values[1::2, :2]])\n",
            "        target = raw_df.values[1::2, 2]\n",
            "\n",
            "    Alternative datasets include the California housing dataset (i.e.\n",
            "    :func:`~sklearn.datasets.fetch_california_housing`) and the Ames housing\n",
            "    dataset. You can load the datasets as follows::\n",
            "\n",
            "        from sklearn.datasets import fetch_california_housing\n",
            "        housing = fetch_california_housing()\n",
            "\n",
            "    for the California housing dataset and::\n",
            "\n",
            "        from sklearn.datasets import fetch_openml\n",
            "        housing = fetch_openml(name=\"house_prices\", as_frame=True)\n",
            "\n",
            "    for the Ames housing dataset.\n",
            "    \n",
            "  warnings.warn(msg, category=FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "kTzuqxwi8mMH",
        "outputId": "6d3b281e-bfe4-419e-f15b-c74df5ccd8e8"
      },
      "source": [
        "import pandas as pd\n",
        "bos = pd.DataFrame(boston.data, columns = boston.feature_names)\n",
        "bos['Price'] = boston.target\n",
        "bos.head()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>CRIM</th>\n",
              "      <th>ZN</th>\n",
              "      <th>INDUS</th>\n",
              "      <th>CHAS</th>\n",
              "      <th>NOX</th>\n",
              "      <th>RM</th>\n",
              "      <th>AGE</th>\n",
              "      <th>DIS</th>\n",
              "      <th>RAD</th>\n",
              "      <th>TAX</th>\n",
              "      <th>PTRATIO</th>\n",
              "      <th>B</th>\n",
              "      <th>LSTAT</th>\n",
              "      <th>Price</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.00632</td>\n",
              "      <td>18.0</td>\n",
              "      <td>2.31</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.538</td>\n",
              "      <td>6.575</td>\n",
              "      <td>65.2</td>\n",
              "      <td>4.0900</td>\n",
              "      <td>1.0</td>\n",
              "      <td>296.0</td>\n",
              "      <td>15.3</td>\n",
              "      <td>396.90</td>\n",
              "      <td>4.98</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.02731</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.07</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.469</td>\n",
              "      <td>6.421</td>\n",
              "      <td>78.9</td>\n",
              "      <td>4.9671</td>\n",
              "      <td>2.0</td>\n",
              "      <td>242.0</td>\n",
              "      <td>17.8</td>\n",
              "      <td>396.90</td>\n",
              "      <td>9.14</td>\n",
              "      <td>21.6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.02729</td>\n",
              "      <td>0.0</td>\n",
              "      <td>7.07</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.469</td>\n",
              "      <td>7.185</td>\n",
              "      <td>61.1</td>\n",
              "      <td>4.9671</td>\n",
              "      <td>2.0</td>\n",
              "      <td>242.0</td>\n",
              "      <td>17.8</td>\n",
              "      <td>392.83</td>\n",
              "      <td>4.03</td>\n",
              "      <td>34.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.03237</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.18</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.458</td>\n",
              "      <td>6.998</td>\n",
              "      <td>45.8</td>\n",
              "      <td>6.0622</td>\n",
              "      <td>3.0</td>\n",
              "      <td>222.0</td>\n",
              "      <td>18.7</td>\n",
              "      <td>394.63</td>\n",
              "      <td>2.94</td>\n",
              "      <td>33.4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.06905</td>\n",
              "      <td>0.0</td>\n",
              "      <td>2.18</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.458</td>\n",
              "      <td>7.147</td>\n",
              "      <td>54.2</td>\n",
              "      <td>6.0622</td>\n",
              "      <td>3.0</td>\n",
              "      <td>222.0</td>\n",
              "      <td>18.7</td>\n",
              "      <td>396.90</td>\n",
              "      <td>5.33</td>\n",
              "      <td>36.2</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      CRIM    ZN  INDUS  CHAS    NOX  ...    TAX  PTRATIO       B  LSTAT  Price\n",
              "0  0.00632  18.0   2.31   0.0  0.538  ...  296.0     15.3  396.90   4.98   24.0\n",
              "1  0.02731   0.0   7.07   0.0  0.469  ...  242.0     17.8  396.90   9.14   21.6\n",
              "2  0.02729   0.0   7.07   0.0  0.469  ...  242.0     17.8  392.83   4.03   34.7\n",
              "3  0.03237   0.0   2.18   0.0  0.458  ...  222.0     18.7  394.63   2.94   33.4\n",
              "4  0.06905   0.0   2.18   0.0  0.458  ...  222.0     18.7  396.90   5.33   36.2\n",
              "\n",
              "[5 rows x 14 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv7him32AUNa"
      },
      "source": [
        "def forward_selection(data, target, significance_level=0.05):\n",
        "    initial_features = data.columns.tolist()\n",
        "    best_features = []\n",
        "    while (len(initial_features)>0):\n",
        "        remaining_features = list(set(initial_features)-set(best_features))\n",
        "        new_pval = pd.Series(index=remaining_features)\n",
        "        for new_column in remaining_features:\n",
        "            model = sm.OLS(target, sm.add_constant(data[best_features+[new_column]])).fit()\n",
        "            new_pval[new_column] = model.pvalues[new_column]\n",
        "        min_p_value = new_pval.min()\n",
        "        if(min_p_value<significance_level):\n",
        "            best_features.append(new_pval.idxmin())\n",
        "        else:\n",
        "            break\n",
        "    return best_features"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BKn0MRs-AV9e",
        "outputId": "cc5af882-5b5d-4fec-c0f0-e90984bb4669"
      },
      "source": [
        "import statsmodels.api as sm\n",
        "X = bos.drop(\"Price\", 1)       # feature matrix\n",
        "y = bos['Price']               # target feature\n",
        "forward_selection(X,y)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:6: DeprecationWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n",
            "  \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['LSTAT',\n",
              " 'RM',\n",
              " 'PTRATIO',\n",
              " 'DIS',\n",
              " 'NOX',\n",
              " 'CHAS',\n",
              " 'B',\n",
              " 'ZN',\n",
              " 'CRIM',\n",
              " 'RAD',\n",
              " 'TAX']"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    }
  ]
}