Model Construction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use the generated feature file as the input for the model construction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>phone</th>\n",
       "      <th>search_1</th>\n",
       "      <th>search_7</th>\n",
       "      <th>search_14</th>\n",
       "      <th>play_1</th>\n",
       "      <th>play_7</th>\n",
       "      <th>play_14</th>\n",
       "      <th>down_1</th>\n",
       "      <th>down_7</th>\n",
       "      <th>down_14</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>167591585</td>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>168134533</td>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>36</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>167891787</td>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>44</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id phone  search_1  search_7  search_14  play_1  play_7  play_14  \\\n",
       "0  167591585    ar         0         0          0       0       0        0   \n",
       "1  168134533    ar         0         4         10      22      36       70   \n",
       "2  167891787    ar         0         0          0      28      44       71   \n",
       "\n",
       "   down_1  down_7  down_14  churn  \n",
       "0       0       0        0      1  \n",
       "1       0       0        1      0  \n",
       "2       0       0        0      0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('generate.csv')\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>phone</th>\n",
       "      <th>search_1</th>\n",
       "      <th>search_7</th>\n",
       "      <th>search_14</th>\n",
       "      <th>play_1</th>\n",
       "      <th>play_7</th>\n",
       "      <th>play_14</th>\n",
       "      <th>down_1</th>\n",
       "      <th>down_7</th>\n",
       "      <th>down_14</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>36</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ar</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>44</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  phone  search_1  search_7  search_14  play_1  play_7  play_14  down_1  \\\n",
       "0    ar         0         0          0       0       0        0       0   \n",
       "1    ar         0         4         10      22      36       70       0   \n",
       "2    ar         0         0          0      28      44       71       0   \n",
       "\n",
       "   down_7  down_14  churn  \n",
       "0       0        0      1  \n",
       "1       0        1      0  \n",
       "2       0        0      0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_feature = df.drop('user_id', axis = 1)\n",
    "df_feature.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>search_1</th>\n",
       "      <th>search_7</th>\n",
       "      <th>search_14</th>\n",
       "      <th>play_1</th>\n",
       "      <th>play_7</th>\n",
       "      <th>play_14</th>\n",
       "      <th>down_1</th>\n",
       "      <th>down_7</th>\n",
       "      <th>down_14</th>\n",
       "      <th>churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>36</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>44</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   search_1  search_7  search_14  play_1  play_7  play_14  down_1  down_7  \\\n",
       "0         0         0          0       0       0        0       0       0   \n",
       "1         0         4         10      22      36       70       0       0   \n",
       "2         0         0          0      28      44       71       0       0   \n",
       "\n",
       "   down_14  churn  \n",
       "0        0      1  \n",
       "1        1      0  \n",
       "2        0      0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_phone = pd.get_dummies(df_feature['phone'])\n",
    "del df_feature['phone']\n",
    "df_feature.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>search_1</th>\n",
       "      <th>search_7</th>\n",
       "      <th>search_14</th>\n",
       "      <th>play_1</th>\n",
       "      <th>play_7</th>\n",
       "      <th>play_14</th>\n",
       "      <th>down_1</th>\n",
       "      <th>down_7</th>\n",
       "      <th>down_14</th>\n",
       "      <th>churn</th>\n",
       "      <th>ar</th>\n",
       "      <th>ip</th>\n",
       "      <th>mc</th>\n",
       "      <th>wp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>36</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>44</td>\n",
       "      <td>71</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   search_1  search_7  search_14  play_1  play_7  play_14  down_1  down_7  \\\n",
       "0         0         0          0       0       0        0       0       0   \n",
       "1         0         4         10      22      36       70       0       0   \n",
       "2         0         0          0      28      44       71       0       0   \n",
       "\n",
       "   down_14  churn  ar  ip  mc  wp  \n",
       "0        0      1   1   0   0   0  \n",
       "1        1      0   1   0   0   0  \n",
       "2        0      0   1   0   0   0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_matrix = df_feature.join(df_phone)\n",
    "df_matrix.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y = np.array(df_matrix['churn'])\n",
    "del df_matrix['churn']\n",
    "x = np.array(df_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "shape of x is:(238828, 13)\n",
      "shape of y is:(238828,)\n"
     ]
    }
   ],
   "source": [
    "print('shape of x is:%s' % str(x.shape))\n",
    "print('shape of y is:%s' % str(y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(x,y,test_size=0.5, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13,)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_names = df_matrix.columns.values\n",
    "feature_names.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV  \n",
    "from sklearn import cross_validation\n",
    "\n",
    "LogisticModel = LogisticRegression()\n",
    "LogisticModel.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.747425\n",
      "Testing DataSet Score: 0.747551\n"
     ]
    }
   ],
   "source": [
    "print('Training DataSet Score: %f' % LogisticModel.score(x_train, y_train))\n",
    "print('Testing DataSet Score: %f' % LogisticModel.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using GridSearch method to find the best model parameters with lowest mean cross validation score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression: Best parameters set found on development set:\n",
      "{'penalty': 'l1', 'C': 8, 'tol': 0.0001}\n"
     ]
    }
   ],
   "source": [
    "LR_param = {\n",
    "    \"penalty\": [\"l1\", \"l2\"],\n",
    "    \"C\": [0.5,1,2,4,8,16],\n",
    "    \"tol\": [0.001,0.0001,0.00001]\n",
    "}\n",
    "LR = LogisticRegression(n_jobs = 4)\n",
    "LR_grid = GridSearchCV(LR, LR_param, cv=5)\n",
    "LR_grid.fit(x_train, y_train)\n",
    "print(\"Logistic Regression: Best parameters set found on development set:\")  \n",
    "print(LR_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.747358\n",
      "Testing DataSet Score: 0.747484\n"
     ]
    }
   ],
   "source": [
    "LogisticModel = LogisticRegression(penalty = 'l1', C = 8, tol = 0.0001)\n",
    "LogisticModel.fit(x_train, y_train)\n",
    "print('Training DataSet Score: %f' % LogisticModel.score(x_train, y_train))\n",
    "print('Testing DataSet Score: %f' % LogisticModel.score(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_top_values(lst, n, labels):\n",
    "    return [labels[i] for i in np.argsort(lst)[::-1][:n]]  # np.argsort by default sorts values in ascending order"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The top 5 features to determine the churn activities are:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ar', 'ip', 'search_1', 'play_1', 'search_14']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coeff = np.absolute(LogisticModel.coef_[0])\n",
    "get_top_values(coeff, 5, feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            n_estimators=10, n_jobs=1, oob_score=False, random_state=None,\n",
       "            verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RandomForestModel = RandomForestClassifier()\n",
    "RandomForestModel.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.864455\n",
      "Testing DataSet Score: 0.791289\n"
     ]
    }
   ],
   "source": [
    "print(\"Training DataSet Score: %f\" % (RandomForestModel.score(x_train, y_train)))\n",
    "print(\"Testing DataSet Score: %f\" % (RandomForestModel.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest: Best parameters set found on development set:\n",
      "{'min_samples_split': 100, 'n_estimators': 60, 'criterion': 'gini'}\n"
     ]
    }
   ],
   "source": [
    "RF_param = {\n",
    "    \"n_estimators\": [60,70,80,90],\n",
    "    \"criterion\": [\"entropy\",\"gini\"],\n",
    "    \"min_samples_split\": [80,100,120,140,160],\n",
    "}\n",
    "RF = RandomForestClassifier(n_jobs=-1, random_state = 1)\n",
    "RF_grid = GridSearchCV(RF, RF_param, cv=5)\n",
    "RF_grid.fit(x_train, y_train)\n",
    "print(\"Random Forest: Best parameters set found on development set:\")  \n",
    "print(RF_grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.816244\n",
      "Testing DataSet Score: 0.803474\n"
     ]
    }
   ],
   "source": [
    "RandomForestModel = RandomForestClassifier(min_samples_split=100, n_estimators=60, criterion='gini')\n",
    "RandomForestModel.fit(x_train, y_train)\n",
    "print(\"Training DataSet Score: %f\" % (RandomForestModel.score(x_train, y_train)))\n",
    "print(\"Testing DataSet Score: %f\" % (RandomForestModel.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The top 5 features to determine the churn activities are:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['play_7', 'play_14', 'search_14', 'play_1', 'search_7']"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_top_values(RandomForestModel.feature_importances_, 5, feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier  #GBM algorithm\n",
    "from sklearn import cross_validation, metrics   #Additional scklearn functions\n",
    "import matplotlib.pylab as plt\n",
    "%matplotlib inline\n",
    "from matplotlib.pylab import rcParams\n",
    "rcParams['figure.figsize'] = 12, 4\n",
    "def modelfit(alg, x_train, y_train, names, performCV=True, printFeatureImportance=True, cv_folds=5):\n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(x_train, y_train)\n",
    "        \n",
    "    #Predict training set:\n",
    "    y_predictions = alg.predict(x_train)\n",
    "    y_predprob = alg.predict_proba(x_train)[:,1]\n",
    "    \n",
    "    #Perform cross-validation:\n",
    "    if performCV:\n",
    "        cv_score = cross_validation.cross_val_score(alg, x_train, y_train, cv=cv_folds, scoring='roc_auc')\n",
    "    \n",
    "    #Print model report:\n",
    "    print \"\\nModel Report\"\n",
    "    print \"Accuracy : %.4g\" % metrics.accuracy_score(y_train, y_predictions)\n",
    "    print \"AUC Score (Train): %f\" % metrics.roc_auc_score(y_train, y_predprob)\n",
    "    \n",
    "    if performCV:\n",
    "        print \"CV Score : Mean - %.7g | Std - %.7g | Min - %.7g | Max - %.7g\" % (np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score))\n",
    "        \n",
    "    #Print Feature Importance:\n",
    "    if printFeatureImportance:\n",
    "        feat_imp = pd.Series(alg.feature_importances_, names).sort_values(ascending=False)\n",
    "        feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "        plt.ylabel('Feature Importance Score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.8052\n",
      "AUC Score (Train): 0.875286\n",
      "CV Score : Mean - 0.8742115 | Std - 0.001526648 | Min - 0.8725417 | Max - 0.8770997\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAExCAYAAABChQXvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYLWV57/3vD1RAEUTBgcmNiAMogtmAcR5AQQ5iFBWj\nBhEleuAYX2Ne0RgHHILG+L4x4jmiojiC4rQNIBIH1CiRLSCTEplkiBEUEAIIbLjPH6salm0PtXt3\n9Vqr9vdzXevqVU9Nd3VVV9/rWc/zVKoKSZIkSYtrnVEHIEmSJPWRibYkSZLUARNtSZIkqQMm2pIk\nSVIHTLQlSZKkDphoS5IkSR0w0ZYkSZI6YKItSUCSS5PcnOS/h16br+E2n5rkisWKseU+P5nkXUu5\nz9kkeXuSz4w6DkkaFRNtSbrLPlW14dDrP0cZTJK7jXL/a2KSY5ekxWKiLUnzSPK4JD9Mcl2SnyZ5\n6tC8A5P8LMkNSS5O8pdN+b2Ak4DNh2vIp9c4T6/1bmrW35jkbODGJHdr1vtSkquTXJLktS3jXpak\nmhgvT3Jtklcn2SXJ2c3xfGho+Zcn+bckH0ryuyQ/T/KMofmbJ1mR5JokFyZ51dC8tyc5PslnklwP\nvBp4M/Ci5th/Otfva/h3keSvk1yV5FdJDhyav0GSf0zyyya+HyTZoMU5enmzrxua399L2vz+JGlN\nWeMgSXNIsgVwAvAy4BvAM4AvJXlEVV0NXAX8D+Bi4MnASUlOr6ozkuwFfKaqthzaXpvdvhjYG/gN\ncAfwdeBrTfmWwL8muaCqTm55GLsB2zXxrWiOY3fg7sCZSb5YVacOLXs8sCnwPODLSbapqmuAY4Fz\ngc2BRwCnJLmoqr7drLsv8ALgL4D1mm08tKpeOhTLrL+vZv4DgY2BLYA9gOOTfLWqrgXeD+wAPB74\nrybWO+Y6R8BNwAeBXarqgiQPAu7b8vcmSWvEGm1JustXmxrR65J8tSl7KXBiVZ1YVXdU1SnASuDZ\nAFV1QlVdVAOnAt8EnrSGcXywqi6vqpuBXYDNqurwqrq1qi4GPgrsvxrbe2dV/b6qvgncCHy+qq6q\nqiuB7wM7Dy17FfD/V9VtVXUccAGwd5KtgCcAb2y2dRbwMQZJ9ZQfVdVXm9/TzTMF0uL3dRtweLP/\nE4H/Bh6eZB3gFcBfVdWVVXV7Vf2wqm5hnnPE4MPKo5JsUFW/qqrzVuN3J0kLZqItSXd5blXdp3k9\ntyl7MPCCoQT8OuCJwIMAkuyV5LSmOcV1DJK7TdcwjsuH3j+YQfOT4f2/GXjAamzv10Pvb55hesOh\n6Surqoamf8mgBntz4JqqumHavC1miXtGLX5fv62qVUPTNzXxbQqsD1w0w2ZnPUdVdSPwIgZNWX6V\n5ISmpluSOmeiLUlzuxz49FACfp+quldVHZFkPeBLDJo0PKCq7gOcCEy1D6kZtncjcM+h6QfOsMzw\nepcDl0zb/72r6tkzrLcYtsgftm/ZGvjP5nXfJPeeNu/KWeL+o+kWv6+5/Ab4PbDtDPNmPUcAVXVy\nVe3B4MPRzxl8IyBJnTPRlqS5fQbYJ8mzkqybZP2m096WwD0YtEW+GljVtMl+5tC6vwbul2TjobKz\ngGcnuW+SBwKvm2f/PwZuaDpIbtDE8KgkuyzaEf6h+wOvTXL3JC8AHsmgWcblwA+Bv29+BzsCBzH4\n/czm18CyptkHzP/7mlVV3QEcDXyg6ZS5bpI/bZL3Wc9Rkgck2TeDzqm3MGiKcsdq/k4kaUFMtCVp\nDk2CuS+D5hpXM6g9/RtgnaYZxWuBLwDXAn/OoLPh1Lo/Bz4PXNw0adgc+DTwU+BSBu2Tj5tn/7cz\n6Dy4E3AJg5rdjzHoMNiFf2fQcfI3wLuB/arqt828FwPLGNRufwV4W1X96xzb+mLz87dJzpjv99XC\nG4BzgNOBa4D3MjgPs56j5vX6JuZrgKcAr1mNfUrSguUPm+JJktZWSV4OvLKqnjjqWCSpD6zRliRJ\nkjpgoi1JkiR1wKYjkiRJUges0ZYkSZI60JtHsG+66aa1bNmyUYchSZKknvvJT37ym6rabL7lepNo\nL1u2jJUrV446DEmSJPVckl+2Wc6mI5IkSVIHTLQlSZKkDphoS5IkSR0w0ZYkSZI6YKItSZIkdcBE\nW5IkSeqAibYkSZLUARNtSZIkqQMm2pIkSVIHevNkyIVYdtgJS7KfS4/Ye0n2I0mSpPFhjbYkSZLU\nARNtSZIkqQMm2pIkSVIHTLQlSZKkDphoS5IkSR0w0ZYkSZI6YKItSZIkdcBEW5IkSepAp4l2kj2T\nXJDkwiSHzTD/1UnOSXJWkh8k2X5o3pua9S5I8qwu45QkSZIWW2eJdpJ1gSOBvYDtgRcPJ9KNz1XV\no6tqJ+B9wAeadbcH9gd2APYEPtxsT5IkSZoIXdZo7wpcWFUXV9WtwLHAvsMLVNX1Q5P3Aqp5vy9w\nbFXdUlWXABc225MkSZImwt063PYWwOVD01cAu01fKMkhwOuBewBPH1r3tGnrbjHDugcDBwNsvfXW\nixK0JEmStBhG3hmyqo6sqm2BNwJvWc11j6qq5VW1fLPNNusmQEmSJGkBuky0rwS2GpresimbzbHA\ncxe4riRJkjRWuky0Twe2S7JNknsw6Ny4YniBJNsNTe4N/KJ5vwLYP8l6SbYBtgN+3GGskiRJ0qLq\nrI12Va1KcihwMrAucHRVnZfkcGBlVa0ADk2yO3AbcC1wQLPueUm+AJwPrAIOqarbu4pVkiRJWmxd\ndoakqk4ETpxW9tah9381x7rvBt7dXXSSJElSd0beGVKSJEnqIxNtSZIkqQMm2pIkSVIHTLQlSZKk\nDphoS5IkSR0w0ZYkSZI6YKItSZIkdcBEW5IkSeqAibYkSZLUARNtSZIkqQMm2pIkSVIHTLQlSZKk\nDphoS5IkSR0w0ZYkSZI6cLdRB6DFs+ywE5ZkP5cesfeS7EeSJGmSWaMtSZIkdcBEW5IkSepA60Q7\nyT27DESSJEnqk3kT7SSPT3I+8PNm+jFJPtx5ZJIkSdIEa1Oj/f8BzwJ+C1BVPwWe3GVQkiRJ0qRr\n1XSkqi6fVnR7B7FIkiRJvdFmeL/LkzweqCR3B/4K+Fm3YUmSJEmTrU2N9quBQ4AtgCuBnZppSZIk\nSbOYs0Y7ybrAy6rqJQvZeJI9gX8C1gU+VlVHTJv/euCVwCrgauAVVfXLZt7twDnNopdV1XMWEoMk\nSZI0CnPWaFfV7cCfL2TDTZJ+JLAXsD3w4iTbT1vsTGB5Ve0IHA+8b2jezVW1U/MyyZYkSdJEadN0\n5AdJPpTkSUkeO/Vqsd6uwIVVdXFV3QocC+w7vEBVfaeqbmomTwO2XK3oJUmSpDHVpjPkTs3Pw4fK\nCnj6POttAQyPVnIFsNscyx8EnDQ0vX6SlQyalRxRVV+dvkKSg4GDAbbeeut5wpEkSZKWzryJdlU9\nresgkrwUWA48Zaj4wVV1ZZKHAN9Ock5VXTQttqOAowCWL19eXccpSZIktdXmyZAbJ/lAkpXN6x+T\nbNxi21cCWw1Nb9mUTd/+7sDfAs+pqlumyqvqyubnxcB3gZ1b7FOSJEkaC23aaB8N3AC8sHldD3yi\nxXqnA9sl2SbJPYD9gRXDCyTZGfgIgyT7qqHyTZKs17zfFHgCcH6LfUqSJEljoU0b7W2r6vlD0+9I\nctZ8K1XVqiSHAiczGN7v6Ko6L8nhwMqqWgH8A7Ah8MUkcNcwfo8EPpLkDgYfBo6oKhNtSZIkTYw2\nifbNSZ5YVT8ASPIE4OY2G6+qE4ETp5W9dej97rOs90Pg0W32IUmSJI2jNon2a4BjhtplXwu8vLOI\nJEmSpB5oM+rIWcBjkmzUTF/feVSSJEnShGsz6sh7ktynqq6vquubjorvWorgJEmSpEnVZtSRvarq\nuqmJqroWeHZ3IUmSJEmTr02ive7UUHsASTYA1ptjeUmSJGmt16Yz5GeBbyWZGjv7QOCY7kKSJEmS\nJl+bzpDvTfJTYHeggHdW1cmdRyZJkiRNsDY12lTVN5KcDjwZ+E23IUmSJEmTb9Y22kn+JcmjmvcP\nAs4FXgF8Osnrlig+SZIkaSLN1Rlym6o6t3l/IHBKVe0D7MYg4ZYkSZI0i7kS7duG3j+D5lHqVXUD\ncEeXQUmSJEmTbq422pcn+V/AFcBjgW/AncP73X0JYpMkSZIm1lw12gcBOwAvB1409NCaxwGfmG0l\nSZIkSXPUaFfVVcCrZyj/DvCdLoOSJEmSJl2bJ0NKkiRJWk0m2pIkSVIHTLQlSZKkDsybaCd5WJJv\nJTm3md4xyVu6D02SJEmaXG1qtD8KvIlmXO2qOhvYv8ugJEmSpEnXJtG+Z1X9eFrZqi6CkSRJkvqi\nTaL9myTbAgWQZD/gV51GJUmSJE24uZ4MOeUQ4CjgEUmuBC4BXtppVJIkSdKEmzfRrqqLgd2T3AtY\np6pu6D4sSZIkabK1GXXkPUnuU1U3VtUNSTZJ8q6lCE6SJEmaVG3aaO9VVddNTVTVtcCz22w8yZ5J\nLkhyYZLDZpj/+iTnJzm7GULwwUPzDkjyi+Z1QJv9SZIkSeOiTaK9bpL1piaSbACsN8fyU8utCxwJ\n7AVsD7w4yfbTFjsTWF5VOwLHA+9r1r0v8DZgN2BX4G1JNmkRqyRJkjQW2iTanwW+leSgJAcBpwDH\ntFhvV+DCqrq4qm4FjgX2HV6gqr5TVTc1k6cBWzbvnwWcUlXXNDXopwB7ttinJEmSNBbadIZ8b5Kz\ngWc0Re+sqpNbbHsL4PKh6SsY1FDP5iDgpDnW3WL6CkkOBg4G2HrrrVuEJEmSJC2NNsP7UVUncVcS\nvOiSvBRYDjxlddarqqMYDD3I8uXLq4PQJEmSpAVpM+rI85oOib9Lcn2SG5Jc32LbVwJbDU1v2ZRN\n3/7uwN8Cz6mqW1ZnXUmSJGlctWmj/T4GSfDGVbVRVd27qjZqsd7pwHZJtklyD2B/YMXwAkl2Bj7S\nbP+qoVknA89shhLcBHhmUyZJkiRNhDZNR35dVT9b3Q1X1aokhzJIkNcFjq6q85IcDqysqhXAPwAb\nAl9MAnBZVT2nqq5J8k4GyTrA4VV1zerGIEmSJI1Km0R7ZZLjgK8CU007qKovz7diVZ0InDit7K1D\n73efY92jgaNbxCdJkiSNnTaJ9kbATQyab0wpYN5EW5IkSVpbtRne78ClCESSJEnqk3kT7STrMxjj\negdg/anyqnpFh3FJkiRJE63NqCOfBh7I4GmNpzIYau+GLoOSJEmSJl2bRPuhVfV3wI1VdQywN3M/\n4VGSJEla67VJtG9rfl6X5FHAxsD9uwtJkiRJmnxtRh05qnlozFsYPHBmQ+DvOo1KkiRJmnBtEu1v\nVdW1wPeAhwAk2abTqCRJkqQJ16bpyJdmKDt+sQORJEmS+mTWGu0kj2AwpN/GSZ43NGsjhob5kyRJ\nkvTH5mo68nDgfwD3AfYZKr8BeFWXQUmSJEmTbtZEu6q+luRfgDdW1XuWMCZJkiRp4s3ZRruqbgee\nu0SxSJIkSb3RZtSRf0vyIeA44Mapwqo6o7OoJEmSpAnXJtHeqfl5+FBZAU9f/HAkSZKkfpg30a6q\npy1FIJIkSVKfzDuOdpKNk3wgycrm9Y9JNl6K4CRJkqRJ1eaBNUczGNLvhc3reuATXQYlSZIkTbo2\nbbS3rarnD02/I8lZXQUkSZIk9UGbGu2bkzxxaiLJE4CbuwtJkiRJmnxtarRfAxzTtMsOcA1wQKdR\nSZIkSROuzagjZwGPSbJRM31951FJkiRJE67NqCP3S/JB4LvAd5L8U5L7dR6ZJEmSNMHatNE+Frga\neD6wX/P+uDYbT7JnkguSXJjksBnmPznJGUlWJdlv2rzbk5zVvFa02Z8kSZI0Ltq00X5QVb1zaPpd\nSV4030pJ1gWOBPYArgBOT7Kiqs4fWuwy4OXAG2bYxM1VtdMM5ZIkSdLYa1Oj/c0k+ydZp3m9EDi5\nxXq7AhdW1cVVdSuDmvF9hxeoqkur6mzgjtWOXJIkSRpjbRLtVwGfA25tXscCf5nkhiRzdYzcArh8\naPqKpqyt9ZsnUZ6W5LkzLZDk4KknVl599dWrsWlJkiSpW21GHbn3UgQygwdX1ZVJHgJ8O8k5VXXR\ntNiOAo4CWL58eY0iSEmSJGkmbdpok2RHYNnw8lX15XlWuxLYamh6y6aslaq6svl5cZLvAjsDF825\nkiRJkjQm5k20kxwN7Aicx11tqQuYL9E+HdguyTYMEuz9gT9vE1SSTYCbquqWJJsCTwDe12ZdSZIk\naRy0qdF+XFVtv7obrqpVSQ5l0HFyXeDoqjovyeHAyqpakWQX4CvAJsA+Sd5RVTsAjwQ+kuQOBu3I\nj5g2WokkSZI01tok2j9Ksv1CEt2qOhE4cVrZW4fen86gScn09X4IPHp19ydJkiSNizaJ9qcYJNv/\nBdwCBKiq2rHTyCRJkqQJ1ibR/jjwMuAcHO9akiRJaqVNon11VfkIdEmSJGk1tEm0z0zyOeDrDJqO\nAK2G95MkSZLWWm0S7Q0YJNjPHCprM7yfJEmStNZq82TIA5ciEEmSJKlPZk20k/wzg5rrGVXVazuJ\nSJIkSeqBuWq0Vy5ZFJIkSVLPzJpoV9UxSxmINGzZYScsyX4uPWLvJdmPJEla+6wz6gAkSZKkPjLR\nliRJkjpgoi1JkiR1YN5EO8nDknwrybnN9I5J3tJ9aJIkSdLkalOj/VHgTcBtAFV1NrB/l0FJkiRJ\nk65Non3PqvrxtLJVXQQjSZIk9UWbRPs3SbaleXhNkv2AX3UalSRJkjTh5n0EO3AIcBTwiCRXApcA\nL+k0KkmSJGnCzZloJ1kHWF5Vuye5F7BOVd2wNKFJkiRJk2vOpiNVdQfw/zbvbzTJliRJktpp00b7\nX5O8IclWSe479eo8MkmSJGmCtWmj/aLm5yFDZQU8ZPHDkSRJkvph3kS7qrZZikAkSZKkPpk30U7y\nFzOVV9WnFj8cSZIkqR/aNB3ZZej9+sAzgDMAE21JkiRpFvN2hqyq/zX0ehXwWGDDNhtPsmeSC5Jc\nmOSwGeY/OckZSVY1D8IZnndAkl80rwPaHpAkSZI0DtqMOjLdjcC87baTrAscCewFbA+8OMn20xa7\nDHg58Llp694XeBuwG7Ar8LYkmywgVkmSJGkk2rTR/jrN49cZJObbA19sse1dgQur6uJmO8cC+wLn\nTy1QVZc28+6Ytu6zgFOq6ppm/inAnsDnW+xXGjvLDjthSfZz6RF7L8l+JEnS/Nq00X7/0PtVwC+r\n6ooW620BXD40fQWDGuo2Zlp3i+kLJTkYOBhg6623brlpSZIkqXttmo48u6pObV7/VlVXJHlv55G1\nUFVHVdXyqlq+2WabjTocSZIk6U5tEu09Zijbq8V6VwJbDU1v2ZS1sSbrSpIkSSM3a6Kd5DVJzgEe\nnuTsodclwNkttn06sF2SbZLcA9gfWNEyrpOBZybZpOkE+cymTJIkSZoIc7XR/hxwEvD3wPDQfDdM\ndVKcS1WtSnIogwR5XeDoqjovyeHAyqpakWQX4CvAJsA+Sd5RVTtU1TVJ3skgWQc4vM0+JUmSpHEx\na6JdVb8Dfge8GCDJ/Rk8sGbDJBtW1WXzbbyqTgROnFb21qH3pzNoFjLTukcDR7c4BkmSJGnszNtG\nO8k+SX4BXAKcClzKoKZbkiRJ0izadIZ8F/A44D+qahsGj2A/rdOoJEmSpAnXJtG+rap+C6yTZJ2q\n+g6wvOO4JEmSpInW5oE11yXZEPg+8NkkVzF4DLskSZKkWbSp0d4XuAl4HfAN4CJgny6DkiRJkibd\nvDXaVXVjkgcD21XVMUnuyWC4PkmSJEmzaDPqyKuA44GPNEVbAF/tMihJkiRp0rVpOnII8ATgeoCq\n+gVw/y6DkiRJkiZdm0T7lqq6dWoiyd2A6i4kSZIkafK1SbRPTfJmYIMkewBfBL7ebViSJEnSZGuT\naB8GXA2cA/wlg0eqv6XLoCRJkqRJN+uoI0m2rqrLquoO4KPNS5IkSVILc9Vo3zmySJIvLUEskiRJ\nUm/MlWhn6P1Dug5EkiRJ6pO5Eu2a5b0kSZKkecz1ZMjHJLmeQc32Bs17mumqqo06j06SJEmaULMm\n2lXlY9YlSZKkBWozvJ8kSZKk1TRX0xFJ+iPLDjthSfZz6RF7L8l+JEnqijXakiRJUgdMtCVJkqQO\nmGhLkiRJHTDRliRJkjrQaaKdZM8kFyS5MMlhM8xfL8lxzfx/T7KsKV+W5OYkZzWv/9NlnJIkSdJi\n62zUkSTrAkcCewBXAKcnWVFV5w8tdhBwbVU9NMn+wHuBFzXzLqqqnbqKT5IkSepSlzXauwIXVtXF\nVXUrcCyw77Rl9gWOad4fDzwjSTqMSZIkSVoSXSbaWwCXD01f0ZTNuExVrQJ+B9yvmbdNkjOTnJrk\nSTPtIMnBSVYmWXn11VcvbvSSJEnSGhjXzpC/Arauqp2B1wOfS7LR9IWq6qiqWl5VyzfbbLMlD1KS\nJEmaTZeJ9pXAVkPTWzZlMy6T5G7AxsBvq+qWqvotQFX9BLgIeFiHsUqSJEmLqstE+3RguyTbJLkH\nsD+wYtoyK4ADmvf7Ad+uqkqyWdOZkiQPAbYDLu4wVkmSJGlRdTbqSFWtSnIocDKwLnB0VZ2X5HBg\nZVWtAD4OfDrJhcA1DJJxgCcDhye5DbgDeHVVXdNVrJIkSdJi6yzRBqiqE4ETp5W9dej974EXzLDe\nl4AvdRmbJEmS1KVx7QwpSZIkTTQTbUmSJKkDJtqSJElSB0y0JUmSpA6YaEuSJEkd6HTUEUkad8sO\nO2FJ9nPpEXsvyX4kSePDGm1JkiSpA9ZoS1JPWDsvSePFGm1JkiSpAybakiRJUgdsOiJJGks2hZE0\n6azRliRJkjpgoi1JkiR1wERbkiRJ6oCJtiRJktQBO0NKktQxO3ZKaydrtCVJkqQOmGhLkiRJHTDR\nliRJkjpgoi1JkiR1wERbkiRJ6oCjjkiSpNXiKCpSO9ZoS5IkSR0w0ZYkSZI60GminWTPJBckuTDJ\nYTPMXy/Jcc38f0+ybGjem5ryC5I8q8s4JUmSpMXWWRvtJOsCRwJ7AFcApydZUVXnDy12EHBtVT00\nyf7Ae4EXJdke2B/YAdgc+NckD6uq27uKV5IkrX1sb64udVmjvStwYVVdXFW3AscC+05bZl/gmOb9\n8cAzkqQpP7aqbqmqS4ALm+1JkiRJEyFV1c2Gk/2APavqlc30y4DdqurQoWXObZa5opm+CNgNeDtw\nWlV9pin/OHBSVR0/bR8HAwc3kw8HLujkYP7QpsBvlmA/S6VPx9OnY4F+HU+fjgX6dTx9Ohbo1/H0\n6VigX8fTp2OBfh3PUh3Lg6tqs/kWmujh/arqKOCopdxnkpVVtXwp99mlPh1Pn44F+nU8fToW6Nfx\n9OlYoF/H06djgX4dT5+OBfp1PON2LF02HbkS2GpoesumbMZlktwN2Bj4bct1JUmSpLHVZaJ9OrBd\nkm2S3INB58YV05ZZARzQvN8P+HYN2rKsAPZvRiXZBtgO+HGHsUqSJEmLqrOmI1W1KsmhwMnAusDR\nVXVeksOBlVW1Avg48OkkFwLXMEjGaZb7AnA+sAo4ZIxGHFnSpipLoE/H06djgX4dT5+OBfp1PH06\nFujX8fTpWKBfx9OnY4F+Hc9YHUtnnSElSZKktZlPhpQkSZI6YKItSZIkdcBEW5IkSeqAifZaLslj\nRx2DJKlfkqyT5PGjjkMaNRPteST5QJInjDqOxZDksdNefwKsSLJzHxLuJO8ZdQxdSPLWUcewppL8\nx6hjWKgkz0my/qjj0NySPK+5X/9jkj8bdTxru6q6Azhy1HF0Lckeo45hIZLcL8k/JzkjyU+S/FOS\n+406roVK8mdJNh6avk+S544ypimOOjKPJFcDvwQ2A44DPl9VZ442qoVJcgdwGnDLUPHjmrKqqqeP\nJLAFSPLB6UXAy4BPAVTVa5c8qI4kuayqth51HG0luQGYurGk+XlP4CYG19lGIwlsgZLcDNwInAR8\nHjh5jIYbXS1JrgG+zOA4pp5bMPGSfBh4KIPjAngRcFFVHTK6qBZPknOq6tGjjmN1JXk/8CPgy325\n1qabtPvzlCSnAN8DPtMUvQR4alXtPrqoFi7JWVW107SyM6tq51HFdGccPb32F83UiUryMAY37/0Z\njAv+eQZJ98TU1CV5PvBa4IiqOqkpu6SqthltZKsvyeXAqcA3uSuZez/wBoCqOmZEoS1IkutnmwVs\nUFWdjXm/2JoPQfcB/qaqft2UTeR1BoN7APB0Bg/V2h94FPAVBn//p44yttWV5ALgn4EXA8uA4xkc\nx2mjjGtNJfk58MipZC7JOsB5VfXI0UbWXpLnzTYL+D9VtdlSxrMYmg/d92LwPIzfMziWSfywPf1h\ne3fOAp5eVfdayngWQ5Jzq+pR08om8gMdQJKzq2rHaWVjcTwm2vNIckZVPXZa2Y4M/lG9oKoeOprI\nFibJhsA7GTzW/q+B71bVQ0Yb1epLcm8Gx3F/4A1V9Z9JLp7EY4FBrQiwy1RiOm3e5VW11QjCWrCm\nWdI/AF8FPgRcOMHn5g/uAUkeCLyQwT1gy0k6N8PHkmRrBh8c9mfwwejYqnrzKONbqCT/wuDBZr9s\nph8MfKiq9hltZO0luQ34LHd9GzRsv6q69xKHtCiS3JfB053vbH41gR9QrwVeCvz39FnAcVX1gKWP\nas0k+QCDJ25/oSnaD9i1qt4wuqgWLsnRwHXc1VzpUGCTqnr5yIJqmGjPY1y+elhsSXYGPgDsUFX3\nH3U8C9UkdO8HTgAOraplo41oYZK8C1hRVT+eYd57q+qNIwhrjTS1iocCLwC2rarNRxzSgsx1D0jy\n4KnkbhLMdixJHgG8qKreMYKw1liSU4FdGCQOBewKrAR+B1BVzxlddO0k+QlwQFWdO8O8ifuwDZDk\nlcBfMajYOYtBU8UfVtUzRhrYakpyEvC+qvrODPO+V1VPHkFYa6T5tuGewB0MPjCsw6CJHEzmtw73\nAv4OmLq2TgHeVVU3jS6qARPteSTZsKqmf4qdabkdquq8pYhpsSQJcO+qmq3ZwkRojuN/An9aVS8d\ndTxdmtBEYCaiAAALjUlEQVTr7EHAzlV14qhjWYgkT62q7446jsWQ5ANV9fpRx7HYkjxlrvmTUIOa\n5EnAL6vqshnmLa+qlSMIa40kOYfBB6DTqmqn5gPde6pqtmYyWiJJPsOgjfb3q+pno45nTSVZDvwt\ngyZxU00ta3pzklEw0V4kMzUxmSRJ3lpVh486Ds2tB9fZHlV1yqjj6EKSH1XVn446jsWQ5E1V9fej\njkN/bJLOTZLTq2qXJGcBu1XVLUnOq6odRh1bFybpHpDkacCTmte2wBkMku5/GmlgC9T0P3kDcC6D\nWnoAxuEbR4f3WzyZf5Gx9spRB7BYmlqUvpr06+zjow6gQ30aAvAFow6gjSQ/aH7ekOT6odcNc3Qw\nnnQTcW4aVyS5D4O+Gqck+RqDUbz6amLuAU0zmHczaG7xUQbfPLxmpEGtmaur6utVdUlV/XLqNeqg\n4K7qda25sf9qYL6RLZYyljU1Tw/9By5lLEtsEq6zuXroT+w4rS2M/blZDRPxga6qntj8nMiOggs0\nEecGoKqmxjN/e5LvABsD3xhhSF2bmHtAkm8xGBHmR8D3GXTGv2q0Ua2RtyX5GPAthoYwrqovjy6k\nARPttct1zDGyxQjiWRPHMXsP/YmpVeipJzF7D/1dlz4cLcDEJAxroYk8N5PQTn4tczbwJwyGK/0d\ncF3T9OXm0Ya1YAcCjwDuzl1NR4rBcwNGykR78dw66gBa+BTwYOCPEm3gc0scy5o6G3j/LD30J3LA\n/ZYm4To7Dbhppn+sTTu6vpqYmsYW+nQsfeO5GV8Tc26q6v+BO4fKfTnwCQbfBq83wrDWxC5V9fBR\nBzET22i3lOTLSfZuhiz7I1X1uKWOaXVV1VtmGj6umXfn8HFJJqGjyuuA2ZrCTOzjl3tyne010zBY\nzbyJGwZrNbxs1AEsoi+OOgDNynMzvibmHpDk0CTHAWcC+wJHA3uNNqo18sMk2486iJk46khLTS3p\ngQzGAf0i8Imq6mXt3KSPbDFsknrow1p3nU1MD324s1/Aexk8JClM6FPuAJJsBryKPxwKi6p6xahi\n0oDnZnz17B7wBgZts39SVatGHc+aSvIzBqOnXMKgjfbUuXF4v0mTZGMGT4T7W+ByBr11P1NVt400\nsEXUp4f0TOqHBq+z8ZPkQmCfnow5+0Oaf7LA7VPlVfWlkQUlwHMzzvp0D+ib5mmwf2QcRh6xjfZq\nSHI/Bp28Xsbg65bPAk8EDgCeOrrIFl2fPn1NTJu5KV5nY+vXPfoHe89JfNroWsJzM776dA/olXFI\nqGdjot1Skq8ADwc+zeAT7a+aWcclmbgndq1FJiqZ8zobayubNo1fZcyGj1qAf0ny7El9WmfPeW7G\nV5/uAVoiNh1pKcnTZuvg1TdJTpuETndtTGDzhLXpOpu0c/OJGYprktrOJrmBwYfPMBhD9xbgNia4\nrWlfeG7GXx/uAVp6JtqrIcmjgO0ZGqe5qj41uogWJsmXGTyh76SqumO+5SdZkjdX1XtGHcfq6Mt1\nNp8kj5ppeMZxlWT9qvr9qOOQNBreA7QQJtotJXkbg/ax2wMnMhgG5wdVtd8o41qIPo1s0bce+j27\nznrTQx/u7Aj1awYd1b7P4Lz8brRRLUySPwO+PRV/85jsp1bVV0cbmTw346tP9wAtHRPtlpKcAzwG\nOLOqHpPkAQxGgdhjxKEtWB9GtuhbD/0+XWd97KGfZGsGT758AvBs4Lqq2mm0Ua2+JGdNj3vSmvL0\nledmvPXlHqClY2fI9m6uqjuSrEqyEXAVsNWog1qoHo1s0bce+n26znrVQz/Jlgz+uT6JwYeh84Af\njDSohZvpgUj+PxgPnpsx1bN7gJaIf7ztrWy+wvsog9rT/wZ+NNqQFqZnI1v0rYd+b64z+tdD/zLg\ndOA9VfXqUQezhlYm+QBwZDN9CIPrTaPnuRlffboHaInYdGQBkiwDNqqqs0ccyoL0YWSLtaGHfg+u\ns1710E/yGAbf+jwZ2Br4BXBqVX18pIEtQJJ7AX8H7M7g7+gU4N1VdeNIA5PnZoz16R6gpWOiPY8k\ncz5VsKrOWKpYFtPaMrLFpOjjddbHHvpJNmTwj/ZJDJpeUVUzPpFsXCVZF3hvVb1h1LHoD3luxl8f\n7gFaWiba80gyXPM7/Muaqjl9+hKHtMZ6NrJFL3ro9/Q661UP/aZZ1XrAVAfc74/z08jm0qex8vvG\nczO++nQP0NIx0W4pyQbA/2TwSbYY/JH970mssevZyBa96qHfp+sM+tVDP8lmVXX1qONYDEn+N7AF\ng+E972ySMMHt53vDczO++nQP0NKxM2R7xwDXAx9spv8c+BTwwpFFtHB9Gtmibz30e3Od9bCH/q1N\nJ7UnN9OnAodPaC39+sBvgeFvSgowmRs9z8346tM9QEvEGu2WkpxfVdvPVzYJknwYeDOwP/DXDEa2\nOKuqDhxpYAuQ5GjgOv6wh/59q+rlIwtqDfTsOruDu3rof23U8aypJF8CzmXwYQgGQ2M+pqqeN7qo\nJC0V7wFaCBPtlpJ8BvhQVZ3WTO8GHFJVfzHayNZMD0a26FUP/T5dZ33roT9LM6U/KpsESdYHDgJ2\n4A87RE/kiDB94rkZX326B2jpTPJX7EvtT4AfJrmsmd4auKBp71xVtePoQmtnrpEtkjx20ka2aHro\nv6NnPfQn/jqbUlU/TXIRcBF39dB/CjCRiTZwc5InVtUPAJI8Abh5xDEt1KeBnwPPAg4HXgL05uFC\nE85zM776dA/QErFGu6Ukcw7fMwk9j3s6skWveuj34Tqb0rce+k0N/aeAjZuia4EDJvHboKkOw0nO\nrqodk9ydwfnpzd/SpPLcjK8+3QO0dKzRbmmSE4QpVfU0mH1kixGGtibOTLKCnvTQ78N1NmSvPvTQ\nT/L6oclPMXhAEgyut92BSfwne1vz87pmTP3/Au4/wnh0F8/NmOnpPUBLxER77dSbkS2wh/4460sP\n/Xs3Px8O7AJ8jcG3QC8FfjyqoNbQUUk2YdC/YQWwIfDW0Yakhudm/PTxHqAlYtORtVCfRrbQ+Opb\nD/0k3wP2rqobmul7AydU1ZPnXlNSH3gP0EJYo712OiPJ46aNbLFyxDEtiD30x9q2VfX8oel3JDlr\nZNGsuQcAtw5N39qUTZzmIVXvATavqr2SbA/86aSOCNMnnpux1pt7gJbOTA/7UP9NjWxxaZJLgR8B\nuyQ5J8mktTX7NPBABj30TwW2BG4YaUSacnOSJ05N9KCH/qeAHyd5e5K3A/8OfHKkES3cJ4GTgc2b\n6f8AXjeyaDTsk3huxlWf7gFaIjYdWQv1bGQLe+iPqT720G+GyHxSM/m9qjpzlPEsVJLTq2qXqb+f\npszxgMeA52a89eUeoKVj05G10CQl0i3YQ3/M9LmHfjPW/ESNNz+LG5Pcj2aYzySPAyatk2pfeW7G\nWI/uAVoiJtqadPbQHz/20B9/r2fw97Jtkn8DNgP2G21IanhupB4x0dZEq6qPNW9PBR4yylg0UFXv\ngDt76D92qIf+24ETRhia7rItsBewFfB8YDf8fzAuPDdSj9gZUhMtyQOSfDzJSc309kkOGnVcAuyh\nP87+rqquBzYBngZ8mMl9aFXfeG6kHjHR1qT7JPbQH1f20B9ftzc/9wY+WlUnAPcYYTy6i+dG6hET\nbU26TavqC8AdAFW1irv+UWmEqurdwIEMRhu5Fjiwqv5+tFGpcWWSjwAvAk5Msh7+PxgXnhupR2z3\npUlnD/0xZg/9sfVCYE/g/VV1XZIHAX8z4pg04LmResRxtDXRmjFN/xl4FIPHfW8G7DfJYzVLkqR+\n8OsoTbqpHvqPZ9BW+xf4TY0kSRoDJtqadPbQlyRJY8lEW5POHvqSJGksmWhr0tlDX5IkjSU7Q2qi\nJbkngx7651TVL5oe+o+uqm+OODRJkrSWM9GWJEmSOuBX7JIkSVIHTLQlSZKkDphoS5IkSR0w0ZYk\nSZI68H8BOrAzDzax2woAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1175e5b90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(random_state=1)\n",
    "modelfit(gbm, x_train, y_train, feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.805241\n",
      "Testing DataSet Score: 0.801782\n"
     ]
    }
   ],
   "source": [
    "print(\"Training DataSet Score: %f\" % (gbm.score(x_train, y_train)))\n",
    "print(\"Testing DataSet Score: %f\" % (gbm.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=8,\n",
       "              max_features='sqrt', max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=50,\n",
       "              min_samples_split=500, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=10,\n",
       "              subsample=0.8, verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=False, n_jobs=4,\n",
       "       param_grid={'n_estimators': [70, 80, 90, 100]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {'n_estimators':range(70, 101 ,10)}\n",
    "gsearch1 = GridSearchCV(\n",
    "    estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=500,min_samples_leaf=50,max_depth=8,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "    param_grid = param_test1, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch1.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.87549, std: 0.00147, params: {'n_estimators': 70}, mean: 0.87554, std: 0.00145, params: {'n_estimators': 80}, mean: 0.87554, std: 0.00144, params: {'n_estimators': 90}, mean: 0.87557, std: 0.00145, params: {'n_estimators': 100}]\n",
      "Best Parameter: {'n_estimators': 100}\n",
      "Best Score: 0.875566\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(gsearch1.grid_scores_)\n",
    "print('Best Parameter: %s' % gsearch1.best_params_)\n",
    "print('Best Score: %f' % gsearch1.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "              max_features='sqrt', max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "              min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=10,\n",
       "              subsample=0.8, verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=False, n_jobs=4,\n",
       "       param_grid={'min_samples_split': [200, 400, 600, 800, 1000], 'max_depth': [5, 7, 9, 11, 13, 15]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test2 = {'max_depth':range(5,16,2), 'min_samples_split':range(200,1001,200)}\n",
    "gsearch2 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=100, max_features='sqrt', subsample=0.8, random_state=10), \n",
    "param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch2.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.87474, std: 0.00157, params: {'min_samples_split': 200, 'max_depth': 5}, mean: 0.87468, std: 0.00160, params: {'min_samples_split': 400, 'max_depth': 5}, mean: 0.87467, std: 0.00148, params: {'min_samples_split': 600, 'max_depth': 5}, mean: 0.87475, std: 0.00158, params: {'min_samples_split': 800, 'max_depth': 5}, mean: 0.87476, std: 0.00155, params: {'min_samples_split': 1000, 'max_depth': 5}, mean: 0.87528, std: 0.00162, params: {'min_samples_split': 200, 'max_depth': 7}, mean: 0.87532, std: 0.00150, params: {'min_samples_split': 400, 'max_depth': 7}, mean: 0.87533, std: 0.00148, params: {'min_samples_split': 600, 'max_depth': 7}, mean: 0.87529, std: 0.00149, params: {'min_samples_split': 800, 'max_depth': 7}, mean: 0.87519, std: 0.00152, params: {'min_samples_split': 1000, 'max_depth': 7}, mean: 0.87529, std: 0.00142, params: {'min_samples_split': 200, 'max_depth': 9}, mean: 0.87549, std: 0.00152, params: {'min_samples_split': 400, 'max_depth': 9}, mean: 0.87546, std: 0.00150, params: {'min_samples_split': 600, 'max_depth': 9}, mean: 0.87566, std: 0.00150, params: {'min_samples_split': 800, 'max_depth': 9}, mean: 0.87553, std: 0.00154, params: {'min_samples_split': 1000, 'max_depth': 9}, mean: 0.87484, std: 0.00141, params: {'min_samples_split': 200, 'max_depth': 11}, mean: 0.87521, std: 0.00141, params: {'min_samples_split': 400, 'max_depth': 11}, mean: 0.87545, std: 0.00141, params: {'min_samples_split': 600, 'max_depth': 11}, mean: 0.87559, std: 0.00155, params: {'min_samples_split': 800, 'max_depth': 11}, mean: 0.87551, std: 0.00157, params: {'min_samples_split': 1000, 'max_depth': 11}, mean: 0.87459, std: 0.00147, params: {'min_samples_split': 200, 'max_depth': 13}, mean: 0.87508, std: 0.00146, params: {'min_samples_split': 400, 'max_depth': 13}, mean: 0.87539, std: 0.00155, params: {'min_samples_split': 600, 'max_depth': 13}, mean: 0.87528, std: 0.00144, params: {'min_samples_split': 800, 'max_depth': 13}, mean: 0.87532, std: 0.00157, params: {'min_samples_split': 1000, 'max_depth': 13}, mean: 0.87414, std: 0.00128, params: {'min_samples_split': 200, 'max_depth': 15}, mean: 0.87485, std: 0.00146, params: {'min_samples_split': 400, 'max_depth': 15}, mean: 0.87517, std: 0.00141, params: {'min_samples_split': 600, 'max_depth': 15}, mean: 0.87517, std: 0.00147, params: {'min_samples_split': 800, 'max_depth': 15}, mean: 0.87536, std: 0.00143, params: {'min_samples_split': 1000, 'max_depth': 15}]\n",
      "Best Parameter: {'min_samples_split': 800, 'max_depth': 9}\n",
      "Best Score: 0.875663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(gsearch2.grid_scores_)\n",
    "print('Best Parameter: %s' % gsearch2.best_params_)\n",
    "print('Best Score: %f' % gsearch2.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=9,\n",
       "              max_features='sqrt', max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "              min_samples_split=800, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=10,\n",
       "              subsample=0.8, verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=False, n_jobs=4,\n",
       "       param_grid={'min_samples_leaf': [100, 110, 120, 130, 140, 150]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test3 = {'min_samples_leaf':range(100,151,10)}\n",
    "gsearch3 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=100,max_depth=9,min_samples_split=800,max_features='sqrt', subsample=0.8, random_state=10), \n",
    "param_grid = param_test3, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch3.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.87582, std: 0.00153, params: {'min_samples_leaf': 100}, mean: 0.87580, std: 0.00148, params: {'min_samples_leaf': 110}, mean: 0.87574, std: 0.00149, params: {'min_samples_leaf': 120}, mean: 0.87568, std: 0.00149, params: {'min_samples_leaf': 130}, mean: 0.87575, std: 0.00148, params: {'min_samples_leaf': 140}, mean: 0.87572, std: 0.00150, params: {'min_samples_leaf': 150}]\n",
      "Best Parameter: {'min_samples_leaf': 100}\n",
      "Best Score: 0.875819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(gsearch3.grid_scores_)\n",
    "print('Best Parameter: %s' % gsearch3.best_params_)\n",
    "print('Best Score: %f' % gsearch3.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=9,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=100,\n",
       "              min_samples_split=800, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=10,\n",
       "              subsample=1.0, verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=False, n_jobs=4,\n",
       "       param_grid={'subsample': [0.6, 0.7, 0.75, 0.8, 0.85, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test4 = {'subsample':[0.6,0.7,0.75,0.8,0.85,0.9]}\n",
    "gsearch4 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, n_estimators=100,max_depth=9, min_samples_split=800, min_samples_leaf=100, random_state=10),\n",
    "param_grid = param_test4, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch4.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.87600, std: 0.00147, params: {'subsample': 0.6}, mean: 0.87596, std: 0.00142, params: {'subsample': 0.7}, mean: 0.87595, std: 0.00142, params: {'subsample': 0.75}, mean: 0.87596, std: 0.00146, params: {'subsample': 0.8}, mean: 0.87603, std: 0.00142, params: {'subsample': 0.85}, mean: 0.87595, std: 0.00138, params: {'subsample': 0.9}]\n",
      "Best Parameter: {'subsample': 0.85}\n",
      "Best Score: 0.876027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(gsearch4.grid_scores_)\n",
    "print('Best Parameter: %s' % gsearch4.best_params_)\n",
    "print('Best Score: %f' % gsearch4.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=9,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_split=1e-07, min_samples_leaf=100,\n",
       "              min_samples_split=800, min_weight_fraction_leaf=0.0,\n",
       "              n_estimators=100, presort='auto', random_state=10,\n",
       "              subsample=0.85, verbose=0, warm_start=False),\n",
       "       fit_params={}, iid=False, n_jobs=4,\n",
       "       param_grid={'learning_rate': [0.1, 0.05, 0.01, 0.005, 0.001]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='roc_auc', verbose=0)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test5 = {'learning_rate':[0.1,0.05,0.01,0.005,0.001]}\n",
    "gsearch5 = GridSearchCV(estimator = GradientBoostingClassifier(n_estimators=100,max_depth=9, min_samples_split=800, min_samples_leaf=100, subsample=0.85,random_state=10),\n",
    "param_grid = param_test5, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch5.fit(x_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[mean: 0.87603, std: 0.00142, params: {'learning_rate': 0.1}, mean: 0.87609, std: 0.00149, params: {'learning_rate': 0.05}, mean: 0.87570, std: 0.00153, params: {'learning_rate': 0.01}, mean: 0.87551, std: 0.00152, params: {'learning_rate': 0.005}, mean: 0.87522, std: 0.00155, params: {'learning_rate': 0.001}]\n",
      "Best Parameter: {'learning_rate': 0.05}\n",
      "Best Score: 0.876086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "print(gsearch5.grid_scores_)\n",
    "print('Best Parameter: %s' % gsearch5.best_params_)\n",
    "print('Best Score: %f' % gsearch5.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.811\n",
      "AUC Score (Train): 0.879490\n",
      "CV Score : Mean - 0.8760861 | Std - 0.00148567 | Min - 0.8741274 | Max - 0.8786741\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtQAAAExCAYAAABcTDVcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8ZGV95/HPF1R2QaVd2GxEokEGtwaJW1wgARkgURSM\nGiEo0cCoY8yIRo2iUckYZmIkmaCi4AaKBtvQiMQYXIm0gCwqEQFZYqQRkA4ii/zmjzoXiutdqm/d\nulXn9Of9etXr1jl1btX33NN1+ldPPc9zUlVIkiRJWpgNxh1AkiRJajMLakmSJGkIFtSSJEnSECyo\nJUmSpCFYUEuSJElDsKCWJEmShmBBLUmSJA3BglrSeiXJVUluS/JffbdthnzOZyW5drEyDviaH03y\nrqV8zdkkeXuSj487hySNiwW1pPXR/lW1ed/tP8YZJsn9xvn6w2hzdklaLBbUktRIsmeSbya5Ocl3\nkzyr77HDknw/ydokVyT542b9ZsCZwDb9Ld7TW5Cnt2I3LeVvTHIRcGuS+zW/99kka5JcmeQ1A+Ze\nnqSajNckuSnJq5LsnuSiZn8+0Lf9oUm+keQDSX6e5AdJntv3+DZJVia5McnlSV7Z99jbk5yW5ONJ\nbgFeBbwZOLjZ9+/O9ffq/1sk+dMk1yf5SZLD+h7fJMlfJ/lxk+/rSTYZ4Bgd2rzW2ubv95JB/n6S\nNCxbFiQJSLItcAbwMuCLwHOBzyZ5bFWtAa4H/jtwBfBM4Mwk51XV+Un2BT5eVdv1Pd8gL/tiYD/g\nBuBu4AvA55v12wH/nOSyqjprwN14CrBzk29lsx97AfcHLkjymao6p2/b04CtgecDn0uyY1XdCJwC\nXAJsAzwWODvJj6rqX5rfPRB4IfCHwEbNczy6ql7al2XWv1fz+MOBLYFtgb2B05KcXlU3Ae8DHgc8\nFfjPJuvdcx0j4BfA+4Hdq+qyJI8AHjzg302ShmILtaT10elNC+fNSU5v1r0UWFVVq6rq7qo6G1gN\nPA+gqs6oqh9VzznAl4BnDJnj/VV1TVXdBuwOLKuqY6rqjqq6AvggcMg6PN87q+qXVfUl4FbgU1V1\nfVVdB3wNeGLfttcD/7eq7qyqU4HLgP2SbA88DXhj81wXAh+iVzxP+VZVnd78nW6bKcgAf687gWOa\n118F/BfwmCQbAH8EvLaqrquqX1XVN6vqduY5RvQ+lOyaZJOq+klVXboOfztJWjALaknro9+rqq2a\n2+816x4JvLCv0L4ZeDrwCIAk+yY5t+kGcTO9Im7rIXNc03f/kfS6jfS//puBh63D8/207/5tMyxv\n3rd8XVVV3/KP6bVIbwPcWFVrpz227Sy5ZzTA3+tnVXVX3/IvmnxbAxsDP5rhaWc9RlV1K3AwvS4o\nP0lyRtNyLUkjZ0EtST3XAB/rK7S3qqrNquq9STYCPkuvK8LDqmorYBUw1a+jZni+W4FN+5YfPsM2\n/b93DXDltNffoqqeN8PvLYZtc99+KTsA/9HcHpxki2mPXTdL7l9bHuDvNZcbgF8CO83w2KzHCKCq\nzqqqvel9CPoBvRZ+SRo5C2pJ6vk4sH+S302yYZKNm8Fz2wEPoNdXeA1wV9Nn+nf6fvenwEOSbNm3\n7kLgeUkenOThwOvmef1vA2ubgYqbNBl2TbL7ou3hfT0UeE2S+yd5IfCb9LpTXAN8E3hP8zfYDTic\n3t9nNj8FljfdNWD+v9esqupu4ETguGZw5IZJfqsp0mc9RkkeluTA9AaJ3k6vC8nd6/g3kaQFsaCW\nJKApJA+k181iDb3W0D8DNmi6P7wG+DRwE/AH9Ab9Tf3uD4BPAVc0XRG2AT4GfBe4il7/4VPnef1f\n0RvE9wTgSnottR+iN3BvFP6N3gDGG4C/BA6qqp81j70YWE6vtfofgb+oqn+e47k+0/z8WZLz5/t7\nDeANwMXAecCNwLH0jsOsx6i5vb7JfCPw28Cr1+E1JWnBct8udJKkrktyKPCKqnr6uLNIUhfYQi1J\nkiQNwYJakiRJGoJdPiRJkqQh2EItSZIkDaF1lx7feuuta/ny5eOOIUmSpI77zne+c0NVLZtvu9YV\n1MuXL2f16tXjjiFJkqSOS/LjQbazy4ckSZI0BAtqSZIkaQgW1JIkSdIQLKglSZKkIVhQS5IkSUOw\noJYkSZKGYEEtSZIkDcGCWpIkSRqCBbUkSZI0hNZdKXEhlh99xpK8zlXv3W9JXkeSJEmTwxZqSZIk\naQgW1JIkSdIQLKglSZKkIVhQS5IkSUOwoJYkSZKGYEEtSZIkDcGCWpIkSRqCBbUkSZI0BAtqSZIk\naQgW1JIkSdIQLKglSZKkIVhQS5IkSUOwoJYkSZKGYEEtSZIkDWGkBXWSfZJcluTyJEfP8PihSdYk\nubC5vWKUeSRJkqTFdr9RPXGSDYHjgb2Ba4Hzkqysqu9N2/TUqjpqVDkkSZKkURplC/UewOVVdUVV\n3QGcAhw4wteTJEmSltwoC+ptgWv6lq9t1k33giQXJTktyfYzPVGSI5KsTrJ6zZo1o8gqSZIkLci4\nByV+AVheVbsBZwMnzbRRVZ1QVSuqasWyZcuWNKAkSZI0l1EW1NcB/S3O2zXr7lFVP6uq25vFDwFP\nHmEeSZIkadGNsqA+D9g5yY5JHgAcAqzs3yDJI/oWDwC+P8I8kiRJ0qIb2SwfVXVXkqOAs4ANgROr\n6tIkxwCrq2ol8JokBwB3ATcCh44qjyRJkjQKIyuoAapqFbBq2rq39d1/E/CmUWaQJEmSRmncgxIl\nSZKkVrOgliRJkoZgQS1JkiQNwYJakiRJGoIFtSRJkjQEC2pJkiRpCBbUkiRJ0hAsqCVJkqQhWFBL\nkiRJQ7CgliRJkoZgQS1JkiQNYeCCOsmmowwiSZIktdG8BXWSpyb5HvCDZvnxSf5u5MkkSZKkFhik\nhfr/AL8L/Aygqr4LPHOUoSRJkqS2GKjLR1VdM23Vr0aQRZIkSWqd+w2wzTVJngpUkvsDrwW+P9pY\nkiRJUjsM0kL9KuBIYFvgOuAJzbIkSZK03puzhTrJhsDLquolS5RHkiRJapU5W6ir6lfAHyxRFkmS\nJKl1BulD/fUkHwBOBW6dWllV548slSRJktQSgxTUT2h+HtO3roDnLH4cSZIkqV3mLair6tlLEUSS\nJElqo0GulLhlkuOSrG5uf51ky6UIJ0mSJE26QabNOxFYC7youd0CfGSUoSRJkqS2GKQP9U5V9YK+\n5XckuXBUgSRJkqQ2GaSF+rYkT59aSPI04LbRRZIkSZLaY5AW6lcDJ/X1m74JOHRkiSRJkqQWGWSW\njwuBxyd5YLN8y8hTSZIkSS0xyCwf706yVVXdUlW3JHlQknctRThJkiRp0g3Sh3rfqrp5aqGqbgKe\nN8iTJ9knyWVJLk9y9BzbvSBJJVkxyPNKkiRJk2KQgnrDJBtNLSTZBNhoju2nttsQOB7YF9gFeHGS\nXWbYbgvgtcC/DRpakiRJmhSDFNSfAL6c5PAkhwNnAycN8Ht7AJdX1RVVdQdwCnDgDNu9EzgW+OWA\nmSVJkqSJMW9BXVXHAu8CfhN4LPDOqvqrAZ57W+CavuVrm3X3SPIkYPuqOmOuJ0pyxNSVGtesWTPA\nS0uSJElLY5AWaqrqi8B7gG8CNyzGCyfZADgO+NMBXv+EqlpRVSuWLVu2GC8vSZIkLYpZC+ok/5Rk\n1+b+I4BLgD8CPpbkdQM893XA9n3L2zXrpmwB7Ar8a5KrgD2BlQ5MlCRJUpvM1UK9Y1Vd0tw/DDi7\nqvYHnkKvsJ7PecDOSXZM8gDgEGDl1INV9fOq2rqqllfVcuBc4ICqWr2QHZEkSZLGYa6C+s6++88F\nVgFU1Vrg7vmeuKruAo4CzgK+D3y6qi5NckySAxYeWZIkSZocc10p8Zok/4PeYMInAV+Ee6bNu/8g\nT15Vq2gK8b51b5tl22cN8pySJEnSJJmrhfpw4HHAocDBfRd32RP4yIhzSZIkSa0wawt1VV0PvGqG\n9V8BvjLKUJIkSVJbDDRtniRJkqSZWVBLkiRJQ7CgliRJkoYwb0Gd5DeSfDnJJc3ybkneMvpokiRJ\n0uQbpIX6g8CbaOalrqqL6F2kRZIkSVrvDVJQb1pV35627q5RhJEkSZLaZpCC+oYkOwEFkOQg4Ccj\nTSVJkiS1xFxXSpxyJHAC8Ngk1wFXAi8daSpJkiSpJeYtqKvqCmCvJJsBG1TV2tHHkiRJktphkFk+\n3p1kq6q6tarWJnlQknctRThJkiRp0g3Sh3rfqrp5aqGqbgKeN7pIkiRJUnsMUlBvmGSjqYUkmwAb\nzbG9JEmStN4YZFDiJ4AvJ/lIs3wYcNLoIkmSJEntMcigxGOTXAQ8t1n1zqo6a7SxJEmSpHYYpIWa\nqjoTOHPEWSRJkqTWGWSWj+cn+WGSnye5JcnaJLcsRThJkiRp0g3SQv1XwP5V9f1Rh5EkSZLaZpBZ\nPn5qMS1JkiTNbJAW6tVJTgVOB26fWllVnxtZKkmSJKklBimoHwj8AvidvnUFWFBLkiRpvTfItHmH\nLUUQSZIkqY3mLaiTbAwcDjwO2HhqfVX90QhzSZIkSa0wyKDEjwEPB34XOAfYDlg7ylCSJElSWwxS\nUD+6qt4K3FpVJwH7AU8ZbSxJkiSpHQYpqO9sft6cZFdgS+Cho4skSZIktccgs3yckORBwFuAlcDm\nwFtHmkqSJElqiUEK6i9X1U3AV4FHASTZcaSpJEmSpJYYpMvHZ2dYd9ogT55knySXJbk8ydEzPP6q\nJBcnuTDJ15PsMsjzSpIkSZNi1hbqJI+lN1Xelkme3/fQA+mbPm+O398QOB7YG7gWOC/Jyqr6Xt9m\nn6yq/9dsfwBwHLDPOu+FJEmSNCZzdfl4DPDfga2A/fvWrwVeOcBz7wFcXlVXACQ5BTgQuKegrqpb\n+rbfjN4VGCVJkqTWmLWgrqrPJ/kn4I1V9e4FPPe2wDV9y9cyw3R7SY4EXg88AHjOTE+U5AjgCIAd\ndthhAVEkSZKk0ZizD3VV/Qr4vVEGqKrjq2on4I30ZhKZaZsTqmpFVa1YtmzZKONIkiRJ62SQWT6+\nkeQDwKnArVMrq+r8eX7vOmD7vuXtmnWzOQX4+wHySJIkSRNjkIL6Cc3PY/rWFbN0z+hzHrBzM8Xe\ndcAhwB/0b5Bk56r6YbO4H/BDJEmSpBaZt6Cuqmcv5Imr6q4kRwFnARsCJ1bVpUmOAVZX1UrgqCR7\n0bsa403AyxfyWpIkSdK4zFtQJ9kS+Avgmc2qc4Bjqurn8/1uVa0CVk1b97a++69dp7SSJEnShBnk\nwi4n0psq70XN7RbgI6MMJUmSJLXFIH2od6qqF/QtvyPJhaMKJEmSJLXJIC3UtyV5+tRCkqcBt40u\nkiRJktQeg7RQvxo4qelLHeBGHDwoSZIkAYPN8nEh8PgkD2yWb5nnVyRJkqT1xrxdPpI8JMn7gX8F\nvpLkb5I8ZOTJJEmSpBYYpA/1KcAa4AXAQc39U0cZSpIkSWqLQfpQP6Kq3tm3/K4kB48qkCRJktQm\ng7RQfynJIUk2aG4vonf1Q0mSJGm9N0hB/Urgk8Adze0U4I+TrE3iAEVJkiSt1waZ5WOLpQgiSZIk\ntdEgfahJshuwvH/7qvrciDJJkiRJrTFvQZ3kRGA34FLg7mZ1ARbUkiRJWu8N0kK9Z1XtMvIkkiRJ\nUgsNMijxW0ksqCVJkqQZDNJCfTK9ovo/gduBAFVVu400mSRJktQCgxTUHwZeBlzMvX2oJUmSJDFY\nQb2mqlaOPIkkSZLUQoMU1Bck+STwBXpdPgCnzZMkSZJgsIJ6E3qF9O/0rXPaPEmSJInBrpR42FIE\n0WCWH33GkrzOVe/db0leR5Ikqe1mLaiT/C29lugZVdVrRpJIkiRJapG5WqhXL1kKSZIkqaVmLair\n6qSlDCJJkiS10SBXSpQkSZI0CwtqSZIkaQgW1JIkSdIQ5i2ok/xGki8nuaRZ3i3JW0YfTZIkSZp8\ng7RQfxB4E3AnQFVdBBwyyJMn2SfJZUkuT3L0DI+/Psn3klzUFO2PXJfwkiRJ0rgNUlBvWlXfnrbu\nrvl+KcmGwPHAvsAuwIuT7DJtswuAFVW1G3Aa8FcD5JEkSZImxiAF9Q1JdqK5yEuSg4CfDPB7ewCX\nV9UVVXUHcApwYP8GVfWVqvpFs3gusN3AySVJkqQJMO+lx4EjgROAxya5DrgSeMkAv7ctcE3f8rXA\nU+bY/nDgzJkeSHIEcATADjvsMMBLS5IkSUtjzoI6yQb0umTslWQzYIOqWrvYIZK8FFgB/PZMj1fV\nCfSKelasWDHr5dAlSZKkpTZnl4+quhv4X839W9exmL4O2L5vebtm3X0k2Qv4c+CAqrp9HZ5fkiRJ\nGrtB+lD/c5I3JNk+yYOnbgP83nnAzkl2TPIAejODrOzfIMkTgX+gV0xfv87pJUmSpDEbpA/1wc3P\nI/vWFfCouX6pqu5KchRwFrAhcGJVXZrkGGB1Va0E/jewOfCZJABXV9UB67gPkiRJ0tjMW1BX1Y4L\nffKqWgWsmrbubX3391roc0uSJEmTYN6COskfzrS+qk5e/DiSJElSuwzS5WP3vvsbA88FzgcsqCVJ\nkrTeG6TLx//oX06yFb2LtEiSJEnrvUFm+ZjuVmDB/aolSZKkLhmkD/UXaC47Tq8A3wX4zChDSZIk\nSW0xSB/q9/Xdvwv4cVVdO6I8kiRJUqsM0uXjeVV1TnP7RlVdm+TYkSeTJEmSWmCQgnrvGdbtu9hB\nJEmSpDaatctHklcDfwI8KslFfQ9tAXxj1MEkSZKkNpirD/UngTOB9wBH961fW1U3jjSVJEmS1BKz\nFtRV9XPg58CLAZI8lN6FXTZPsnlVXb00ESVJkqTJNW8f6iT7J/khcCVwDnAVvZZrSZIkab03yKDE\ndwF7Av9eVTvSu/T4uSNNJUmSJLXEIAX1nVX1M2CDJBtU1VeAFSPOJUmSJLXCIBd2uTnJ5sDXgE8k\nuZ7e5cclSZKk9d4gLdQHAr8AXgd8EfgRsP8oQ0mSJEltMW8LdVXdmuSRwM5VdVKSTYENRx9NkiRJ\nmnyDzPLxSuA04B+aVdsCp48ylCRJktQWg3T5OBJ4GnALQFX9EHjoKENJkiRJbTFIQX17Vd0xtZDk\nfkCNLpIkSZLUHoMU1OckeTOwSZK9gc8AXxhtLEmSJKkdBimojwbWABcDfwysAt4yylCSJElSW8w6\ny0eSHarq6qq6G/hgc5MkSZLUZ64W6ntm8kjy2SXIIkmSJLXOXAV1+u4/atRBJEmSpDaaq6CuWe5L\nkiRJasx1pcTHJ7mFXkv1Js19muWqqgeOPJ0kSZI04WYtqKvKy4tLkiRJ8xhk2jxJkiRJsxhpQZ1k\nnySXJbk8ydEzPP7MJOcnuSvJQaPMIkmSJI3CyArqJBsCxwP7ArsAL06yy7TNrgYOBT45qhySJEnS\nKM01KHFYewCXV9UVAElOAQ4Evje1QVVd1Tx29whzSJIkSSMzyi4f2wLX9C1f26xbZ0mOSLI6yeo1\na9YsSjhJkiRpMbRiUGJVnVBVK6pqxbJly8YdR5IkSbrHKAvq64Dt+5a3a9ZJkiRJnTHKgvo8YOck\nOyZ5AHAIsHKErydJkiQtuZEV1FV1F3AUcBbwfeDTVXVpkmOSHACQZPck1wIvBP4hyaWjyiNJkiSN\nwihn+aCqVgGrpq17W9/98+h1BZEkSZJaqRWDEiVJkqRJZUEtSZIkDcGCWpIkSRqCBbUkSZI0BAtq\nSZIkaQgW1JIkSdIQLKglSZKkIVhQS5IkSUMY6YVdpPksP/qMJXmdq96735K8jiRJWv/YQi1JkiQN\nwYJakiRJGoIFtSRJkjQEC2pJkiRpCA5KlBaJAywlSVo/WVBLmpEfECRJGoxdPiRJkqQhWFBLkiRJ\nQ7CgliRJkoZgQS1JkiQNwYJakiRJGoIFtSRJkjQEC2pJkiRpCBbUkiRJ0hC8sIukzvMiNZKkUbKF\nWpIkSRqCLdSS1DK2uEvSZLGFWpIkSRqCBbUkSZI0BAtqSZIkaQgj7UOdZB/gb4ANgQ9V1XunPb4R\ncDLwZOBnwMFVddUoM0mSJstS9Alfqv7gXevf3qVjI43SyFqok2wIHA/sC+wCvDjJLtM2Oxy4qaoe\nDfwf4NhR5ZEkSZJGYZQt1HsAl1fVFQBJTgEOBL7Xt82BwNub+6cBH0iSqqoR5pIkSeuZrn17oMmS\nUdWuSQ4C9qmqVzTLLwOeUlVH9W1zSbPNtc3yj5ptbpj2XEcARzSLjwEuG0no+9oauGHerdqhS/sC\n3dqfLu0LdGt/urQv0K396dK+QLf2p0v7At3any7tCyzd/jyyqpbNt1Er5qGuqhOAE5byNZOsrqoV\nS/mao9KlfYFu7U+X9gW6tT9d2hfo1v50aV+gW/vTpX2Bbu1Pl/YFJm9/RjnLx3XA9n3L2zXrZtwm\nyf2ALekNTpQkSZJaYZQF9XnAzkl2TPIA4BBg5bRtVgIvb+4fBPyL/aclSZLUJiPr8lFVdyU5CjiL\n3rR5J1bVpUmOAVZX1Urgw8DHklwO3Eiv6J4US9rFZMS6tC/Qrf3p0r5At/anS/sC3dqfLu0LdGt/\nurQv0K396dK+wITtz8gGJUqSJEnrA6+UKEmSJA3BglqSJEkaggW1JEmSNAQL6vVEkieNO4MkqVuS\nbJDkqePOIY2bBTWQ5LgkTxt3jsWS5EnTbk8GViZ5YhcK6yTvHneGxZLk38edYRSSvG3cGdZFkgOS\nbDzuHJpbkuc35+u/TvL7484jqKq7gePHnWPUkuw97gwLkeQhSf42yflJvpPkb5I8ZNy5FirJ7yfZ\nsm95qyS/N85MU5zlA0iyBvgxsAw4FfhUVV0w3lQLl+Ru4Fzg9r7VezbrqqqeM5ZgC5Dk/dNXAS8D\nTgaoqtcseagFSrIWmHrDpfm5KfALesflgWMJNgJJrq6qHcadY1BJbgNuBc4EPgWcVVW/Gm+qhUty\nI/A5evvSifn9k/wd8Gh6+wRwMPCjqjpyfKkWV5KLq+q/jTvHukryPuBbwOe68G9tJm07p01Jcjbw\nVeDjzaqXAM+qqr3Gl2rhklxYVU+Ytu6CqnriuDLdk6Oj//bXydTBSPIb9E7Sh9CbO/tT9IrrVrUi\nJnkB8BrgvVV1ZrPuyqracbzJ1l2Sa4BzgC9xbxH6PuANAFV10piirbPmw8FWwJ9V1U+bda08LgBJ\nbpntIWCTqhrZPPeLLckFwHPoXWDqEGBX4B/pvf/PGWe2hUhyGfC3wIuB5cBp9Pbl3HHmGkaSHwC/\nOVWwJdkAuLSqfnO8ydZNkufP9hDw/6pq2VLmWQxNY8FmwF3AL+ntS+saCZJMv/jcPQ8Bz6mqzZYy\nz2JIcklV7TptXSs/uAEkuaiqdpu2biL2x4IaSHJ+VT1p2rrd6P1n9MKqevR4ki1cks2Bd9K75Puf\nAv9aVY8ab6p1l2QLevvxUOANVfUfSa5o474ANN1v/jdwOvAB4PIW78vVwO5THw6mPXZNVW0/hlgL\nMv0ckOThwIvonQO2a9O+wH33J8kO9D4kHELvA90pVfXmceZbiCT/BBxZVT9ulh8JfKCq9h9vsnWT\n5E7gE9z7bVW/g6pqiyWOtCiSPBjYGbin61TbPowmuQl4KfBf0x8CTq2qhy19quEkOQ74NvDpZtVB\nwB5V9YbxpVq4JCcCN3NvN6OjgAdV1aFjC9WwoGZyvi4YhSRPBI4DHldVDx13noVqCtH3AWcAR1XV\n8vEmWrimZe0o4IXATlW1zZgjLUiSdwErq+rbMzx2bFW9cQyxFmSuc0CSR04VcW0x2/4keSxwcFW9\nYwyxhpLkHGB3esVBAXsAq4GfA1TVAeNLN7gk3wFeXlWXzPBYqz6ITknyCuC19BpwLqTXxfCbVfXc\nsQZbR0nOBP6qqr4yw2NfrapnjiHWUJpvDzYF7qb3wWADet3boJ3fImwGvBWY+rd1NvCuqvrF+FL1\nWFDTa82tqumfSGfa7nFVdelSZFpMSQJsUVWzfUXfCs1+/AnwW1X10nHnGVaSRwBPrKpV484ySm14\n3yR5VlX967hzLJYkx1XV68edYzEl+e25Hm9La2iSZwA/rqqrZ3hsRVWtHkOsoSS5mN6HnXOr6gnN\nB7d3V9Vs3Vu0RJJ8nF4f6q9V1ffHnWdYSVYAf06vK9tUt8Ka3g1kHCyo18FMXUPaJsnbquqYcefQ\nfSXZu6rOHneOUejC+2ZKkm9V1W+NO8diSfKmqnrPuHPo17Xp2CQ5r6p2T3Ih8JSquj3JpVX1uHFn\nG4U2nQeSPBt4RnPbCTifXnH9N2MNtkDN+JA3AJfQa3UHYBK+RXTavHWT+TeZeK8Yd4DF0rSKdMWH\nxx1ghLrwvpnStan1XjjuAPNJ8vXm59okt/Td1s4xMLYLJv7Y9Lk2yVb0xoacneTz9GbO6qrWnAea\n7it/Sa+bxAfpfZPw6rGGGs6aqvpCVV1ZVT+euo07FNzbXK7BtKI5f77ZF5Yyy7DmGRH/8KXMMqx5\nRpC3dl7QAbTifTOgLu0LtODDTlU9vfnZysF6Q5j4YzOlqqbmBH97kq8AWwJfHGOkUWvNeSDJl+nN\nwPIt4Gv0BpJfP95UQ/mLJB8Cvkzf1MBV9bnxReqxoO6mm5lj9oUx5BnGqcw+Ir41rQSNZzD7CPI9\nlj6O1J7CYD3UymPTlr7s65GLgCfTmwr058DNTZeV28Yba8EOAx4L3J97u3wUvXn3x8qCet3cMe4A\nAzoZeCTwawU18MklzjKsi4D3zTIivm0T058L/GKm/3CafmFd1Zb3zSBa02o4oK7tT5d4bCZXa45N\nVf1PuGcK2kOBj9D7dnejMcYaxu5V9Zhxh5iJfaj7JPlckv2aac1+TVXtudSZFqKq3jLTVGbNY/dM\nZZakDQNGXgfM1oWlVZcerqp9Z5qOqXmsddMxTenK+2ZALxt3gEX2mXEH0Kw8NpOrNeeBJEclORW4\nADgQOBHYd7yphvLNJLuMO8RMnOWjT9PieRi9OTQ/A3ykqjrbctix2RdaMyJ+Pm0aQQ7det80ffaP\npXchodDSK75NSbIMeCX3nWKKqvqjcWVSj8dmcnXpPJDkDfT6Tn+nqu4ad55hJfk+vdlKrqTXh3rq\n2Dht3iTxfGCzAAAIUElEQVRKsiW9K6T9OXANvZGxH6+qO8cabJF16YI2Hftw0Mrj0oX3TZLLgf27\nMF8rQJJv0vxnCvxqan1VfXZsoQR4bCZZ184DXdJcIfXXTMJMH/ahnibJQ+gNHHsZva9IPgE8HXg5\n8KzxJRuJLn2aak2ftgG07rh06H3z0479J7ppm65YuZ7x2Eyurp0HOmMSCufZWFD3SfKPwGOAj9H7\ndPqT5qFTk7Tu6lXrmdYVoV3RsffN6qa/4elM2JRMC/RPSZ7X9atxtpTHZnJ17TygJWCXjz5Jnj3b\noLEuSnJuVwaMtbWbxEzati9det8k+cgMq6tt/VqTrKX3ITP05qC9HbiTFvcF7QqPzeTrynlAS8uC\nepokuwK70DfHcVWdPL5EC5fkc/SuwHdmVd093/ZtluTNVfXucedYDEl2nWmawEnWlfdNko2r6pfj\nziFpfDwPaCEsqPsk+Qt6/T13AVbRm1rm61V10DhzLVTHZl/ozIj4Lo0gh269b5rBSD+lN1jsa/T2\n4+fjTbVwSX4f+JepfWguD/2sqjp9vMnksZlcXTsPaGlYUPdJcjHweOCCqnp8kofRm6Vg7zFHG0pH\nZl/ozIj4ro0g79r7JskO9K5q+TTgecDNVfWE8aZamCQXTs/eti5FXeWxmWxdOg9oaTgo8b5uq6q7\nk9yV5IHA9cD24w41jA7NvtClEfFdG0HemfdNku3o/Qf6DHofEi4Fvj7WUMOZ6WI7nvcng8dmQnXw\nPKAl4Jv3vlY3X7t9kF5L6H8B3xpvpIXr2OwLXRoR37UR5F1631wNnAe8u6peNe4wi2B1kuOA45vl\nI+kdI42fx2Zyde08oCVgl49ZJFkOPLCqLhpzlAXrwuwLXRwR3+UR5G1/3yR5PL1vcJ4J7AD8EDin\nqj481mALlGQz4K3AXvTeR2cDf1lVt441mDw2E6xr5wEtDQtqIMmcV9irqvOXKsti68rsC13SlRHk\nXX3fJNmc3n+mz6DXXYqqmvHqXJMsyYbAsVX1hnFn0X15bCZfV84DWjoW1ECS/lbc/j/IVCvoc5Y4\n0qLo2OwLnRkR35UR5F183zRdoTYCpgbBfm2Sr8w1ny7NNd81HpvJ1bXzgJaGBXWfJJsAf0LvU2nR\neyP9fVtbE7s0+0LXRsR3aQR5l943SZZV1Zpx51gsSf4e2JbetJn3dCVocX/9zvDYTK6unQe0NByU\neF8nAbcA72+W/wA4GXjR2BINpzOzL9ChEfEdHEHepffNHc1AsWc2y+cAx7TxG4TGxsDPgP5vCwqw\naBs/j83k6tp5QEvAFuo+Sb5XVbvMt64tkvwd8GbgEOBP6c2+cGFVHTbWYAuQ5ETgZu47Iv7BVXXo\n2EItUJK7uXcE+efHnWdYXXrfJPkscAm9DwnQm27y8VX1/PGlkrSUPA9oISyo+yT5OPCBqjq3WX4K\ncGRV/eF4kw2vA7MvdGZEfNdGkHfpfTNL16JfW9cWSTYGDgcex30HJrd+Rpm289hMrq6dB7Q0WvmV\n+Qg9Gfhmkqub5R2Ay5q+yFVVu40v2uDmmn0hyZPaNvtCMyL+HV0ZEV9V303yI+BH3DuC/LeBVhbU\ndOR907gtydOr6usASZ4G3DbmTMP4GPAD4HeBY4CXAF26qFCbeWwmV9fOA1oCtlD3STLnlDhtGeXb\n0dkXOjMivmsjyLvyvoF7vj04GdiyWXUT8PIWf7NzQVU9MclFVbVbkvvT+/fWifdSm3lsJlfXzgNa\nGrZQ92nTf/xzqapnw+yzL4wx2jAuSLKSboyI37dLI8i78L5J8vq+xZPpXUQIev/W9gLa+h/pnc3P\nm5s56f8TeOgY8+heHpsJ0+HzgJaABXW3dWn2hS6NiHcE+eTZovn5GGB34PP0vtF5KfDtcYVaBCck\neRC98Qcrgc2Bt403khoem8nT1fOAloBdPjqsS7MvdIkjyCdXkq8C+1XV2mZ5C+CMqnrm3L8pqSs8\nD2ghbKHutvOT7Dlt9oXVY860IB0bEb9TVb2gb/kdSS4cWxr1exhwR9/yHc26Vmou5vRuYJuq2jfJ\nLsBvtXVGmS7x2Ey0Tp0HtDRmuliGumNq9oWrklwFfAvYPcnFSdrWF+xjwMPpjYg/B9gOWDvWRAt3\nW5KnTy04gnyinAx8O8nbk7wd+Dfgo2NNNJyPAmcB2zTL/w68bmxp1O+jeGwmVdfOA1oCdvnosI7N\nvtCZEfGOIJ9szbSTz2gWv1pVF4wzzzCSnFdVu0+9f5p1zqc7ATw2k61L5wEtDbt8dFibCuYBtH5E\nvCPI26GZp71Vc7XP4dYkD6GZPjPJnoCDXyeDx2aCdew8oCVgQa226MKIeEeQa6m9nt77Zack3wCW\nAQeNN5IaHhupQyyo1QpV9aHm7jnAo8aZZaGq6h1wzwjyJ/WNIH87cMYYo6m7dgL2BbYHXgA8Bc/7\nk8JjI3WIgxLVCkkeluTDSc5slndJcvi4cy2QI8i1VN5aVbcADwKeDfwd7b24U9d4bKQOsaBWW3yU\n7oyIdwS5lsqvmp/7AR+sqjOAB4wxj+7lsZE6xIJabbF1VX0auBugqu7i3v+QWqWq/hI4jN7sHjcB\nh1XVe8abSh11XZJ/AA4GViXZCM/7k8JjI3WI/bXUFp0aEe8Ici2RFwH7AO+rqpuTPAL4szFnUo/H\nRuoQ56FWKzRzgv4tsCu9y3YvAw5y7mZJkjRufr2ktpgaEf9Uen2pf4jfsEiSpAlgQa22cES8JEma\nSBbUagtHxEuSpIlkQa22cES8JEmaSA5KVCsk2ZTeiPiLq+qHzYj4/1ZVXxpzNEmStJ6zoJYkSZKG\n4FfmkiRJ0hAsqCVJkqQhWFBLkiRJQ7CgliRJkobw/wHEjrdArC+vmwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1113294d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gbm = GradientBoostingClassifier(learning_rate = 0.05, n_estimators=100,max_depth=9, min_samples_split=800, min_samples_leaf=100, subsample=0.85,random_state=10)\n",
    "modelfit(gbm, x_train, y_train, feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DataSet Score: 0.810952\n",
      "Testing DataSet Score: 0.804420\n"
     ]
    }
   ],
   "source": [
    "print(\"Training DataSet Score: %f\" % (gbm.score(x_train, y_train)))\n",
    "print(\"Testing DataSet Score: %f\" % (gbm.score(x_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}