Instances pruning with tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import operator\n",
    "import itertools\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "from time import time\n",
    "from scipy.stats import norm\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import scale\n",
    "from hyperopt import STATUS_OK, hp, fmin, tpe, Trials, space_eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Import data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def load_data():\n",
    "    full_data = pd.read_csv(\"Data/X.csv\")\n",
    "    train_y = pd.read_csv(\"Data/y_train.csv\")\n",
    "    # Rename columns to something more interpretable\n",
    "    columns = ([\"reflectance_\" + str(i) for i in range(7)]\n",
    "               + [\"solar_\" + str(i) for i in range(5)] + [\"id\"])\n",
    "    full_data.columns = columns\n",
    "    \n",
    "    # Move ID column to the beginning\n",
    "    id_column = full_data[\"id\"]\n",
    "    full_data.drop(labels=[\"id\"], axis=1, inplace = True)\n",
    "    full_data.insert(0, \"id\", id_column)\n",
    "    \n",
    "    # Add the target value column to the training part\n",
    "    # in full_data\n",
    "    split = 98000\n",
    "    y_id_dict = train_y.set_index(\"id\")[\"y\"].to_dict()\n",
    "    full_data.loc[:(split-1), \"y\"] = full_data.loc[:(split-1), \"id\"].map(y_id_dict)\n",
    "    \n",
    "    # Split into training and testing data\n",
    "    train, test = full_data[:split], full_data[split:]\n",
    "    return (train, test)\n",
    "\n",
    "train, test = load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "random_seed = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "cols_excl = [\"id\", \"y\"]\n",
    "cols_orig = [c for c in train.columns if c not in cols_excl]\n",
    "\n",
    "# Standardise data can make training faster and reduce\n",
    "# the chances of getting stuck in local optima\n",
    "train[cols_orig] = scale(train[cols_orig])\n",
    "test[cols_orig] = scale(test[cols_orig])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Instances pruning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get rid of the noisy instances, we prune a proportion of instances that is the farther from the computed median in each bag, and repeat until the change in the RMSE is below a chosen threshold."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Weight each instance by the gaussian pdf\n",
    "# obtained with the empirical mean and variance\n",
    "def gaussian_weight(data):\n",
    "    cols = list(data.columns)\n",
    "    X = data.copy()\n",
    "    \n",
    "    y_mean_dict = X.groupby(\"id\")[\"y_hat\"].mean().to_dict()\n",
    "    y_std_dict = X.groupby(\"id\")[\"y_hat\"].std().to_dict()\n",
    "    X[\"y_hat_mean\"] = X[\"id\"].map(y_mean_dict)\n",
    "    X[\"y_hat_std\"] = X[\"id\"].map(y_std_dict)\n",
    "    X[\"pdf\"] = norm.pdf(X[\"y_hat\"], X[\"y_hat_mean\"], \n",
    "                        X[\"y_hat_std\"])\n",
    "    y_pdf_sum_dict = X.groupby(\"id\")[\"pdf\"].sum().to_dict()\n",
    "    X[\"pdf_sum\"] = X[\"id\"].map(y_pdf_sum_dict)\n",
    "    X[\"pdf\"] /= X[\"pdf_sum\"]\n",
    "    X[\"y_hat_weighted\"] = X[\"y_hat\"] * X[\"pdf\"]\n",
    "    \n",
    "    y_weighted_dict = X.groupby(\"id\")[\"y_hat_weighted\"].sum().to_dict()\n",
    "    X[\"y_hat_weighted_sum\"] = X[\"id\"].map(y_weighted_dict)\n",
    "    \n",
    "    return(X[cols + [\"y_hat_weighted_sum\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "cols_dnn = cols_orig\n",
    "\n",
    "models_weights = {\"dnn_1\": 1}\n",
    "models_cols = {\"dnn_1\": cols_dnn}\n",
    "\n",
    "# DNNRegressor default learning rate\n",
    "# for Adagrad optimiser\n",
    "learning_rate = 0.1\n",
    "\n",
    "# Only show tensorflow's errors\n",
    "tf.logging.set_verbosity(tf.logging.ERROR)\n",
    "\n",
    "# Scoring function in the hyperopt hyperparameters tuning.\n",
    "def scoring_function(parameters):\n",
    "    print(\"Training the model with parameters: \")\n",
    "    print(parameters)\n",
    "    average_RMSE = 0.0\n",
    "    n_splits = 5\n",
    "    \n",
    "    kf = KFold(n_splits=n_splits, shuffle=True, random_state=random_seed)\n",
    "    nb_fold = 0\n",
    "    for train_index, validation_index in kf.split(train):\n",
    "        nb_fold += 1\n",
    "        train_fold, validation_fold = train.loc[train_index], train.loc[validation_index]\n",
    "        \n",
    "        # Columns used in DNNRegressor\n",
    "        feature_cols = [tf.contrib.layers.real_valued_column(k) for k in cols_dnn]\n",
    "        \n",
    "        # Initialise the number of pruning iteration\n",
    "        pruning_count = 0\n",
    "        # Directory to save the training phase\n",
    "        model_dir = (\"./log_\"\n",
    "                     + str(parameters[\"steps\"]) + \"_\"\n",
    "                     + str(parameters[\"nb_neurons_1\"]) + \"_\"\n",
    "                     + str(nb_fold) + \"_\"\n",
    "                     + str(pruning_count)\n",
    "                    )\n",
    "        model_dnn = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,\n",
    "                                                  hidden_units=[parameters[\"nb_neurons_1\"]],\n",
    "                                                  optimizer=tf.train.ProximalAdagradOptimizer(\n",
    "                                                      learning_rate=learning_rate,\n",
    "                                                      l2_regularization_strength=parameters[\"l2_reg\"]),\n",
    "                                                  dropout=parameters[\"dropout\"],\n",
    "                                                  model_dir=model_dir)\n",
    "\n",
    "        def input_fn(data_set):\n",
    "            feature_cols = {k: tf.constant(data_set[k].values) for k in cols_dnn}\n",
    "            labels = tf.constant(data_set[\"y\"].values)\n",
    "            return feature_cols, labels\n",
    "        \n",
    "        model_dnn.fit(input_fn=lambda: input_fn(train_fold), steps=parameters[\"steps\"])\n",
    "\n",
    "        # Initialise train predictions\n",
    "        train_pred = train_fold[[\"id\"]].assign(y_hat=0)\n",
    "        temp = model_dnn.predict(input_fn=lambda: input_fn(train_fold))\n",
    "        # .predict() returns an iterator; convert to an array\n",
    "        y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "        train_pred[\"y_hat\"] = y_hat\n",
    "\n",
    "        # Use median value by id\n",
    "        y_hat_med = train_pred.groupby(\"id\").median()[\"y_hat\"].to_dict()\n",
    "\n",
    "        RMSE = np.sqrt(mean_squared_error(train_pred[\"id\"].map(y_hat_med).values, train_fold[\"y\"]))\n",
    "        print(\"Pruning {0} RMSE: {1}\".format(pruning_count, RMSE))\n",
    "        \n",
    "        # Recursively prune outliers until the\n",
    "        # decrease in the RMSE is below a threshold\n",
    "        RMSE_decreasing = True\n",
    "        while (RMSE_decreasing):\n",
    "            pruning_count +=1\n",
    "            \n",
    "            train_pred[\"y_med\"] = train_pred[\"id\"].map(y_hat_med)\n",
    "            # Distance from the median for each bag\n",
    "            train_pred[\"score\"] = (train_pred[\"y_hat\"] - train_pred[\"y_med\"])**2\n",
    "            # Rank of each instance by bag\n",
    "            train_pred[\"rank\"] = train_pred.groupby(\"id\")[\"score\"].rank()\n",
    "            bag_size_dict = train_pred.groupby(\"id\")[\"score\"].count().to_dict()\n",
    "            train_pred[\"bag_size\"] = train_pred[\"id\"].map(bag_size_dict)\n",
    "            # Divide rank by the number of instances in each bag\n",
    "            # to obtain a relative rank from 0 to 1\n",
    "            train_pred[\"rank\"] = train_pred[\"rank\"] / train_pred[\"bag_size\"]\n",
    "\n",
    "            # Remove outliers: the instances ranked above \n",
    "            # (1 - parameters[\"outliers_threshold\"])\n",
    "            outliers_index = train_pred[\"rank\"] > (1 - parameters[\"outliers_threshold\"])\n",
    "            train_fold = train_fold.loc[~outliers_index, :].reset_index(drop=True)\n",
    "            \n",
    "            # Train new model on the pruned data\n",
    "            model_dir = (\"./log_\"\n",
    "                         + str(parameters[\"steps\"]) + \"_\"\n",
    "                         + str(parameters[\"nb_neurons_1\"]) + \"_\"\n",
    "                         + str(nb_fold) + \"_\"\n",
    "                         + str(pruning_count)\n",
    "                        )\n",
    "            model_dnn = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,\n",
    "                                                      hidden_units=[parameters[\"nb_neurons_1\"]],\n",
    "                                                      optimizer=tf.train.ProximalAdagradOptimizer(\n",
    "                                                          learning_rate=learning_rate,\n",
    "                                                          l2_regularization_strength=parameters[\"l2_reg\"]),\n",
    "                                                      dropout=parameters[\"dropout\"],\n",
    "                                                      model_dir=model_dir)\n",
    "\n",
    "            model_dnn.fit(input_fn=lambda: input_fn(train_fold), steps=parameters[\"steps\"])\n",
    "            \n",
    "            # Compute new RMSE\n",
    "            train_pred = train_fold[[\"id\"]].assign(y_hat=0)\n",
    "            temp = model_dnn.predict(input_fn=lambda: input_fn(train_fold))\n",
    "            y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "            train_pred[\"y_hat\"] = y_hat\n",
    "\n",
    "            # Use median value by id\n",
    "            y_hat_med = train_pred.groupby(\"id\").median()[\"y_hat\"].to_dict()\n",
    "\n",
    "            new_RMSE = np.sqrt(mean_squared_error(train_pred[\"id\"].map(y_hat_med), train_fold[\"y\"]))\n",
    "            print(\"Pruning {0} RMSE: {1}\".format(pruning_count, new_RMSE))\n",
    "            \n",
    "            if ((RMSE - new_RMSE) > parameters[\"gain_threshold\"]):\n",
    "                RMSE = new_RMSE\n",
    "            else:\n",
    "                RMSE_decreasing = False\n",
    "        \n",
    "        models = {\"dnn_1\": model_dnn}\n",
    "        \n",
    "        # Compute RMSE on validation set\n",
    "        validation_pred = validation_fold[[\"id\", \"y\"]].assign(y_hat=0)\n",
    "        for i, m in models.items():\n",
    "            temp = m.predict(input_fn=lambda: input_fn(validation_fold))\n",
    "            y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "            validation_pred[\"y_hat\"] += models_weights[i] * y_hat\n",
    "\n",
    "        validation_pred = gaussian_weight(validation_pred)\n",
    "        RMSE = np.sqrt(mean_squared_error(validation_pred[\"y_hat_weighted_sum\"], validation_pred[\"y\"]))\n",
    "        \n",
    "        average_RMSE += RMSE\n",
    "        print(\"Validation fold {0} RMSE: {1}\".format(nb_fold, RMSE))\n",
    "\n",
    "    average_RMSE /= n_splits\n",
    "\n",
    "    print(\"Cross-validation score: {0}\\n\".format(average_RMSE))\n",
    "    \n",
    "    return {\"loss\": average_RMSE, \"status\": STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t0 = time()\n",
    "\n",
    "# Grid to pick parameters from.\n",
    "parameters_grid = {\"steps\"             : hp.choice(\"steps\", np.arange(1000, 2000, 100, dtype=int)),\n",
    "                   \"nb_neurons_1\"      : hp.choice(\"nb_neurons_1\", np.arange(8, 12, 1, dtype=int)),\n",
    "                   \"outliers_threshold\": hp.quniform(\"outliers_threshold\", 0.05, 0.051, 0.01),\n",
    "                   \"gain_threshold\"    : hp.quniform(\"gain_threshold\", 0.005, 0.0051, 0.001),\n",
    "                   \"dropout\": hp.quniform(\"dropout\", 0.2, 0.4, 0.1),\n",
    "                   \"l2_reg\": hp.quniform(\"l2_reg\", 0.001, 0.01, 0.001)\n",
    "                  }\n",
    "# Record the information about the cross-validation.\n",
    "trials = Trials()\n",
    "\n",
    "best = fmin(scoring_function, parameters_grid, algo=tpe.suggest, max_evals=1, \n",
    "            trials=trials)\n",
    "\n",
    "computing_time = time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Save the best parameters as a csv.\n",
    "best_parameters = pd.DataFrame({key: [value] for (key, value) in \n",
    "                                zip(space_eval(parameters_grid, best).keys(),\n",
    "                                    space_eval(parameters_grid, best).values())})\n",
    "# Add the corresponding score.\n",
    "best_parameters[\"score\"] = min(trials.losses())\n",
    "best_parameters.to_csv(\"Parameters/best_parameters_6.csv\", encoding=\"utf-8\", index=False)\n",
    "\n",
    "best_parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Training models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruning 0 RMSE: 0.682746000532\n",
      "Pruning 1 RMSE: 0.674742291619\n",
      "Pruning 2 RMSE: 0.677821951825\n"
     ]
    }
   ],
   "source": [
    "cols_dnn = cols_orig\n",
    "models_weights = {\"dnn_1\": 1.0}\n",
    "models_cols = {\"dnn_1\": cols_dnn}\n",
    "best_parameters = pd.read_csv(\"Parameters/best_parameters_6.csv\", encoding=\"utf-8\")\n",
    "\n",
    "feature_cols = [tf.contrib.layers.real_valued_column(k) for k in cols_dnn]\n",
    "\n",
    "# Initialise number of pruning iteration\n",
    "pruning_count = 0\n",
    "model_dir = \"./log_submit_6_\" + str(pruning_count)\n",
    "model_dnn = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,\n",
    "                                          hidden_units=[best_parameters[\"nb_neurons_1\"][0]],\n",
    "                                          model_dir=model_dir)\n",
    "\n",
    "def input_fn(data_set):\n",
    "    feature_cols = {k: tf.constant(data_set[k].values) for k in cols_dnn}\n",
    "    labels = tf.constant(data_set[\"y\"].values)\n",
    "    return feature_cols, labels\n",
    "\n",
    "model_dnn.fit(input_fn=lambda: input_fn(train), steps=best_parameters[\"steps\"][0])\n",
    "\n",
    "# Initialise train predictions\n",
    "train_pred = train[[\"id\"]].assign(y_hat=0)\n",
    "temp = model_dnn.predict(input_fn=lambda: input_fn(train))\n",
    "y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "train_pred[\"y_hat\"] = y_hat\n",
    "\n",
    "# Use median value by id\n",
    "y_hat_med = train_pred.groupby(\"id\").median()[\"y_hat\"].to_dict()\n",
    "\n",
    "RMSE = np.sqrt(mean_squared_error(train_pred[\"id\"].map(y_hat_med).values, train[\"y\"]))\n",
    "print(\"Pruning {0} RMSE: {1}\".format(pruning_count, RMSE))\n",
    "# Prune outliers\n",
    "RMSE_decreasing = True\n",
    "while (RMSE_decreasing):\n",
    "    pruning_count += 1\n",
    "    \n",
    "    train_pred[\"y_med\"] = train_pred[\"id\"].map(y_hat_med)\n",
    "    # Distance from the median for each bag\n",
    "    train_pred[\"score\"] = (train_pred[\"y_hat\"] - train_pred[\"y_med\"])**2\n",
    "    # Rank of each instance by bag\n",
    "    train_pred[\"rank\"] = train_pred.groupby(\"id\")[\"score\"].rank()\n",
    "    bag_size_dict = train_pred.groupby(\"id\")[\"score\"].count().to_dict()\n",
    "    train_pred[\"bag_size\"] = train_pred[\"id\"].map(bag_size_dict)\n",
    "    train_pred[\"rank\"] = train_pred[\"rank\"] / train_pred[\"bag_size\"]\n",
    "\n",
    "    # Remove outliers\n",
    "    outliers_index = train_pred[\"rank\"] > (1 - best_parameters[\"outliers_threshold\"][0])\n",
    "    train = train.loc[~outliers_index, :].reset_index(drop=True)\n",
    "    \n",
    "    model_dir = \"./log_submit_6_\" + str(pruning_count)\n",
    "    model_dnn = tf.contrib.learn.DNNRegressor(feature_columns=feature_cols,\n",
    "                                              hidden_units=[best_parameters[\"nb_neurons_1\"][0]],\n",
    "                                              model_dir=model_dir)\n",
    "\n",
    "    model_dnn.fit(input_fn=lambda: input_fn(train), steps=best_parameters[\"steps\"][0])\n",
    "\n",
    "    # Compute new RMSE\n",
    "    train_pred = train[[\"id\"]].assign(y_hat=0)\n",
    "    temp = model_dnn.predict(input_fn=lambda: input_fn(train))\n",
    "    y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "    train_pred[\"y_hat\"] = y_hat\n",
    "\n",
    "    # Use median value by id\n",
    "    y_hat_med = train_pred.groupby(\"id\").median()[\"y_hat\"].to_dict()\n",
    "\n",
    "    new_RMSE = np.sqrt(mean_squared_error(train_pred[\"id\"].map(y_hat_med), train[\"y\"]))\n",
    "    print(\"Pruning {0} RMSE: {1}\".format(pruning_count, new_RMSE))\n",
    "\n",
    "    if ((RMSE - new_RMSE) > best_parameters[\"gain_threshold\"][0]):\n",
    "        RMSE = new_RMSE\n",
    "    else:\n",
    "        RMSE_decreasing = False\n",
    "\n",
    "models = {\"dnn_1\": model_dnn}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Predicting on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "test_pred = test[[\"id\"]].assign(y_hat=0).reset_index(drop=True)\n",
    "for i, m in models.items():\n",
    "    temp = m.predict(input_fn=lambda: input_fn(test))\n",
    "    y_hat = np.array(list(itertools.islice(temp, 0, None)))\n",
    "    test_pred[\"y_hat\"] += models_weights[i] * y_hat\n",
    "\n",
    "# Use gaussian weights\n",
    "test_pred = gaussian_weight(test_pred)\n",
    "y_hat_by_bag = test_pred.groupby(\"id\").median()[\"y_hat_weighted_sum\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "kaggle_pred = pd.DataFrame({\"Id\": y_hat_by_bag.index, \"y\": y_hat_by_bag.values})\n",
    "kaggle_pred.to_csv(\"Predictions/Prediction_6.csv\", encoding=\"utf-8\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}