01-Ames-Regression-Minimalist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = 'data/train.csv'\n",
    "kaggl_dir  = 'data/test.csv'\n",
    "# submission_path = 'data/test_submission.csv'\n",
    "submission_path = None\n",
    "brute = False\n",
    "interaction_only = True\n",
    "run_lin = True\n",
    "run_ridge = True\n",
    "run_las = True\n",
    "run_elnet = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:49.896529Z",
     "start_time": "2018-05-17T22:50:47.817264Z"
    }
   },
   "outputs": [],
   "source": [
    "# Standard Imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "    \n",
    "# Additional Imports:\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.preprocessing import Imputer, StandardScaler, PolynomialFeatures\n",
    "from sklearn.linear_model import LinearRegression, RidgeCV, LassoCV, Lasso, ElasticNetCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.032759Z",
     "start_time": "2018-05-17T22:50:49.914684Z"
    }
   },
   "outputs": [],
   "source": [
    "train_data = pd.read_csv(train_dir, index_col = 'Id')\n",
    "kaggl_data = pd.read_csv(kaggl_dir,  index_col = 'Id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train/Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.072541Z",
     "start_time": "2018-05-17T22:50:50.064777Z"
    }
   },
   "outputs": [],
   "source": [
    "X = train_data.drop('SalePrice', axis=1)\n",
    "y = train_data['SalePrice']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.092879Z",
     "start_time": "2018-05-17T22:50:50.075551Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.061092Z",
     "start_time": "2018-05-17T22:50:50.048097Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training data has 1640 rows.\n",
      "Testing data has 411 rows.\n",
      "Kaggle data has 879 rows.\n"
     ]
    }
   ],
   "source": [
    "print('Training data has {} rows.'.format(X_train.shape[0]))\n",
    "print('Testing data has {} rows.'.format(X_test.shape[0]))\n",
    "print('Kaggle data has {} rows.'.format(kaggl_data.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Manual Feature Engineering...\n"
     ]
    }
   ],
   "source": [
    "# Manual Feature Engineering\n",
    "print('Manual Feature Engineering...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.104755Z",
     "start_time": "2018-05-17T22:50:50.092879Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Create an 'EDA' dataframe we'll use to do some exploring\n",
    "EDA = X_train.copy()\n",
    "EDA['SalePrice'] = y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.125261Z",
     "start_time": "2018-05-17T22:50:50.108764Z"
    }
   },
   "outputs": [],
   "source": [
    "# There are 27 neighborhoods. Let's put them into groups of 9:\n",
    "neighborhood_ranks = EDA.groupby('Neighborhood')['SalePrice'].mean().sort_values().index\n",
    "\n",
    "low_neigh  = neighborhood_ranks[:9]\n",
    "mid_neigh  = neighborhood_ranks[9:18]\n",
    "high_neigh = neighborhood_ranks[18:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.181334Z",
     "start_time": "2018-05-17T22:50:50.143598Z"
    }
   },
   "outputs": [],
   "source": [
    "def manual_feature_eng(data):\n",
    "    '''Some basic manual feature engineering based on EDA of X_train'''\n",
    "    eng_data = data.copy()\n",
    "    # Years info:\n",
    "    eng_data['Years_Old'] = 2018 - eng_data['Year Built']\n",
    "    eng_data['Garage Age'] = 2018 - eng_data['Garage Yr Blt']\n",
    "    eng_data['Years Since Sale'] = 2018 - eng_data['Yr Sold']\n",
    "    eng_data['Years Since Remodel'] = 2018 - eng_data['Year Remod/Add']\n",
    "    eng_data.drop(['Year Built','Garage Yr Blt','Yr Sold','Year Remod/Add'],\n",
    "                 axis=1, inplace=True)\n",
    "    # Neighborhood info:\n",
    "    eng_data['High_Neigh'] = [1 if x in high_neigh else 0 for x in eng_data['Neighborhood']]\n",
    "    eng_data['Mid_Neigh'] = [1 if x in mid_neigh else 0 for x in eng_data['Neighborhood']]\n",
    "    eng_data['Low_Neigh'] = [1 if x in low_neigh else 0 for x in eng_data['Neighborhood']]\n",
    "    eng_data.drop('Neighborhood', axis=1, inplace=True)\n",
    "    \n",
    "    # Is there miscellaneous furniture?\n",
    "    eng_data['MiscFurn'] = eng_data['Misc Val'] > 0\n",
    "    return eng_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.270813Z",
     "start_time": "2018-05-17T22:50:50.184697Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train = manual_feature_eng(X_train)\n",
    "X_test = manual_feature_eng(X_test)\n",
    "kaggl_data = manual_feature_eng(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing Categorical Data...\n"
     ]
    }
   ],
   "source": [
    "# Data Preprocessing: Categorical Data\n",
    "print('Processing Categorical Data...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.286884Z",
     "start_time": "2018-05-17T22:50:50.279632Z"
    }
   },
   "outputs": [],
   "source": [
    "# Before we begin, let's check to see if there are any columns in the Kaggle \n",
    "# set that aren't in the training set:\n",
    "\n",
    "assert [col for col in kaggl_data.columns if col not in X_train.columns] == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.299854Z",
     "start_time": "2018-05-17T22:50:50.290638Z"
    }
   },
   "outputs": [],
   "source": [
    "# And vice versa:\n",
    "\n",
    "assert [col for col in X_train.columns if col not in kaggl_data.columns] == []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.424678Z",
     "start_time": "2018-05-17T22:50:50.345299Z"
    }
   },
   "outputs": [],
   "source": [
    "# All of our preprocessing will ultimately go here:\n",
    "def preprocessing(data):\n",
    "    try:\n",
    "        cleaned_data = data.drop('PID', axis=1)\n",
    "    except:\n",
    "        cleaned_data = data\n",
    "    fillna_dict = {\n",
    "        'Pool QC':'No Pool',\n",
    "        'Alley':'No Alley',\n",
    "        # Let's let the get_dummies drop 'Misc Features' if NA\n",
    "        'Fence':'No Fence',\n",
    "        'Fireplace Qu':'No Fireplace',\n",
    "        # Lot frontage can be mean imputed\n",
    "        'Garaga Finish': 'No Garage',\n",
    "        'Garage Qual': 'No Garage',\n",
    "        'Garage Cond': 'No Garage',\n",
    "        'Garage Type': 'No Garage',\n",
    "        'Bsmt Exposure':'No Garage',\n",
    "        'BsmtFin Type 2':'No Basement',\n",
    "        'BsmtFin Type 2':'No Basement',\n",
    "        'BsmtFin Type 1':'No Basement',\n",
    "        'Bsmt Cond':'No Basement',\n",
    "        'Bsmt Qual':'No Basement',\n",
    "        'Mas Vnr Type':'No Mas Vnr'        \n",
    "    }\n",
    "    \n",
    "    cleaned_data = cleaned_data.fillna(fillna_dict)\n",
    "    \n",
    "    return(cleaned_data)\n",
    "    \n",
    "X_train = preprocessing(X_train)\n",
    "X_test  = preprocessing(X_test)\n",
    "kaggl_data = preprocessing(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.439282Z",
     "start_time": "2018-05-17T22:50:50.427898Z"
    }
   },
   "outputs": [],
   "source": [
    "# Grab the string columns:\n",
    "string_cols = X_train.select_dtypes(exclude=[np.number]).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.694373Z",
     "start_time": "2018-05-17T22:50:50.442857Z"
    }
   },
   "outputs": [],
   "source": [
    "# Get some dummies:\n",
    "X_train = pd.get_dummies(X_train, columns=string_cols)\n",
    "X_test = pd.get_dummies(X_test, columns=string_cols)\n",
    "kaggl_data = pd.get_dummies(kaggl_data, columns=string_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Addressing column mismatch...\n"
     ]
    }
   ],
   "source": [
    "# Addressing Column Mismatch After Dummifying\n",
    "print('Addressing column mismatch...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.754110Z",
     "start_time": "2018-05-17T22:50:50.697398Z"
    }
   },
   "outputs": [],
   "source": [
    "# Add columns of zeros to test and kaggle sets for columns that *do* appear in\n",
    "# the training set.\n",
    "\n",
    "model_cols = X_train.columns\n",
    "\n",
    "def add_model_cols(data, model_cols):\n",
    "    new_data = data.copy()\n",
    "    for missing_col in [col for col in model_cols if col not in data.columns]:\n",
    "        new_data[missing_col] = 0\n",
    "    return new_data\n",
    "\n",
    "X_test = add_model_cols(X_test, model_cols=model_cols)\n",
    "kaggl_data = add_model_cols(kaggl_data, model_cols=model_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.770118Z",
     "start_time": "2018-05-17T22:50:50.757659Z"
    }
   },
   "outputs": [],
   "source": [
    "# Now, let's only consider columns in X_test and kaggl_data that appear in\n",
    "# the training set. We'll call these 'model columns':\n",
    "\n",
    "kaggl_data = kaggl_data[model_cols]\n",
    "X_test     = X_test[model_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.781986Z",
     "start_time": "2018-05-17T22:50:50.774385Z"
    }
   },
   "outputs": [],
   "source": [
    "# Make sure we've done this correctly:\n",
    "assert X_train.shape[1] == X_test.shape[1] == kaggl_data.shape[1]\n",
    "assert X_train.columns.all() == X_test.columns.all()== kaggl_data.columns.all() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T14:14:47.276989Z",
     "start_time": "2018-05-17T14:14:47.272980Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imputing missing numerical data...\n"
     ]
    }
   ],
   "source": [
    "# Imputing Numerical Missing Data: Handling Numerical Data\n",
    "print('Imputing missing numerical data...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.828133Z",
     "start_time": "2018-05-17T22:50:50.785990Z"
    }
   },
   "outputs": [],
   "source": [
    "imp = Imputer(strategy='mean')\n",
    "imp.fit(X_train)\n",
    "X_train = imp.transform(X_train)\n",
    "X_test  = imp.transform(X_test)\n",
    "kaggl_data = imp.transform(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.836683Z",
     "start_time": "2018-05-17T22:50:50.832279Z"
    }
   },
   "outputs": [],
   "source": [
    "def array_null_check(array):\n",
    "    '''Turns an array into a dataframe so that we can check for null values'''\n",
    "    return pd.DataFrame(array).isnull().sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:50.868097Z",
     "start_time": "2018-05-17T22:50:50.839630Z"
    }
   },
   "outputs": [],
   "source": [
    "assert array_null_check(X_train) == array_null_check(X_test) \\\n",
    "                                 == array_null_check(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Brute Force Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:55.918676Z",
     "start_time": "2018-05-17T22:50:50.871606Z"
    }
   },
   "outputs": [],
   "source": [
    "if brute:\n",
    "    print('Brute force feature engineering...')\n",
    "    pf = PolynomialFeatures(interaction_only=interaction_only)\n",
    "    X_train = pf.fit_transform(X_train)\n",
    "    X_test  = pf.transform(X_test)\n",
    "    kaggl_data = pf.transform(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:55.931478Z",
     "start_time": "2018-05-17T22:50:55.921636Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train has:\n",
      "---1640 rows\n",
      "---275 columns\n"
     ]
    }
   ],
   "source": [
    "# Maybe this is too many columns???\n",
    "print('X_train has:\\n---{} rows\\n---{} columns'.format(X_train.shape[0], X_train.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scaling all columns...\n"
     ]
    }
   ],
   "source": [
    "# Scaling\n",
    "print('Scaling all columns...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:50:59.920683Z",
     "start_time": "2018-05-17T22:50:55.935490Z"
    }
   },
   "outputs": [],
   "source": [
    "ss = StandardScaler()\n",
    "X_train = ss.fit_transform(X_train)\n",
    "X_test  = ss.transform(X_test)\n",
    "kaggl_data = ss.transform(kaggl_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature Elimination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:51:00.811718Z",
     "start_time": "2018-05-17T22:51:00.806521Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold, SelectPercentile, f_regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:51:05.010086Z",
     "start_time": "2018-05-17T22:51:00.814968Z"
    }
   },
   "outputs": [],
   "source": [
    "if brute:\n",
    "    print('Performing automatic feature elimination')\n",
    "    # Only do feature elimination if feature engineering happened by brute force\n",
    "    feature_variances = np.apply_along_axis(np.var, axis=0, arr= X_train)\n",
    "\n",
    "    # Define a percentile threshold. Do I want the top 1% of features by variance?\n",
    "    perc_thresh = np.percentile(feature_variances, 99)\n",
    "    perc_thresh\n",
    "\n",
    "    vt = VarianceThreshold(threshold=perc_thresh)\n",
    "    X_train_reduced = vt.fit_transform(X_train)\n",
    "    X_test_reduced  = vt.transform(X_test)\n",
    "    kaggl_reduced   = vt.transform(kaggl_data)\n",
    "    print('X_train now has:\\n---{} rows\\n---{} columns'.format(X_train.shape[0], X_train.shape[1]))\n",
    "else:\n",
    "    X_train_reduced = X_train\n",
    "    X_test_reduced  = X_test\n",
    "    kaggl_reduced   = kaggl_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:51:05.021878Z",
     "start_time": "2018-05-17T22:51:05.014056Z"
    }
   },
   "outputs": [],
   "source": [
    "# Or do I want to select the top 1% of features according \n",
    "# to the f_regression function?\n",
    "\n",
    "# sp = SelectPercentile(score_func=f_regression, percentile = 1)\n",
    "# X_train_reduced = sp.fit_transform(X_train, y_train)\n",
    "# X_test_reduced  = sp.transform(X_test)\n",
    "# kaggl_reduced   = sp.transform(kaggl_data)\n",
    "# print(X_train.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:51:10.844444Z",
     "start_time": "2018-05-17T22:51:05.025033Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LinearRegression model has average performance of -4.000606872543032e+25\n"
     ]
    }
   ],
   "source": [
    "if run_lin:\n",
    "    lin = LinearRegression()\n",
    "    lin.fit(X_train_reduced, y_train)\n",
    "    cv_scores = cross_val_score(lin, X_train_reduced, y_train, cv=3).mean()\n",
    "\n",
    "    print('{} model has average performance of {}'\n",
    "          .format(str(lin).split('(')[0], cv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T14:27:09.157953Z",
     "start_time": "2018-05-17T14:27:09.154947Z"
    }
   },
   "outputs": [],
   "source": [
    "# Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:51:10.857746Z",
     "start_time": "2018-05-17T22:51:10.847168Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RidgeCV model has average performance of 0.7923282327509859\n"
     ]
    }
   ],
   "source": [
    "if run_ridge:\n",
    "    rid = RidgeCV()\n",
    "    rid.fit(X_train_reduced, y_train)\n",
    "    cv_scores = cross_val_score(rid, X_train_reduced, y_train, cv=3).mean()\n",
    "\n",
    "    print('{} model has average performance of {}'\n",
    "          .format(str(rid).split('(')[0], cv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lasso Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:08.032309Z",
     "start_time": "2018-05-17T22:51:10.860947Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LassoCV model has average performance of 0.8360634781761181\n"
     ]
    }
   ],
   "source": [
    "if run_las:\n",
    "    # Define a reasonable range of alphas based on previous LASSO fits:\n",
    "    alphas = np.logspace(2,4,20)\n",
    "    las = LassoCV(alphas=alphas, n_jobs=-1)\n",
    "    las.fit(X_train_reduced, y_train)\n",
    "    cv_scores = cross_val_score(las, X_train_reduced, y_train, cv=3).mean()\n",
    "    best_alpha = las.alpha_\n",
    "    print('{} model has average performance of {}'\n",
    "          .format(str(las).split('(')[0], cv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:09.643010Z",
     "start_time": "2018-05-17T22:52:08.035584Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lasso model has average performance of 0.840994730059033\n"
     ]
    }
   ],
   "source": [
    "las = Lasso(alpha=best_alpha, max_iter=2000)\n",
    "cv_scores = cross_val_score(las, X_train_reduced, y_train, cv=3).mean()\n",
    "las.fit(X_train_reduced, y_train)\n",
    "print('{} model has average performance of {}'\n",
    "      .format(str(las).split('(')[0], cv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ElasticNet Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:09.659048Z",
     "start_time": "2018-05-17T22:52:09.646405Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ElasticNetCV model has average performance of 0.2989254517143054\n"
     ]
    }
   ],
   "source": [
    "if run_elnet:\n",
    "    elnet = ElasticNetCV(n_alphas=10)\n",
    "    elnet.fit(X_train_reduced, y_train)\n",
    "    cv_scores = cross_val_score(elnet, X_train_reduced, y_train, cv=3).mean()\n",
    "\n",
    "    print('{} model has average performance of {}'\n",
    "          .format(str(elnet).split('(')[0], cv_scores.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Model Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:09.699553Z",
     "start_time": "2018-05-17T22:52:09.661928Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set performance of LinearRegression: -4.096930747827317e+19\n",
      "Test set performance of RidgeCV: 0.921152783117557\n",
      "Test set performance of Lasso: 0.9221014668886816\n",
      "Test set performance of ElasticNetCV: 0.3156100614982734\n"
     ]
    }
   ],
   "source": [
    "models = {}\n",
    "\n",
    "try:\n",
    "    lin_score = lin.score(X_test_reduced, y_test)\n",
    "    models[lin_score] = lin\n",
    "    print('Test set performance of {}: {}'.format(str(lin).split('(')[0],lin_score))\n",
    "except:\n",
    "    pass    \n",
    "\n",
    "try:\n",
    "    rid_score = rid.score(X_test_reduced, y_test)\n",
    "    models[rid_score] = rid\n",
    "    print('Test set performance of {}: {}'.format(str(rid).split('(')[0],rid_score))\n",
    "except:\n",
    "    pass    \n",
    "\n",
    "try:\n",
    "    las_score = las.score(X_test_reduced, y_test)\n",
    "    models[las_score] = las\n",
    "    print('Test set performance of {}: {}'.format(str(las).split('(')[0],las_score))\n",
    "except:\n",
    "    pass          \n",
    "\n",
    "try:\n",
    "    elnet_score = elnet.score(X_test_reduced, y_test)\n",
    "    models[elnet_score] = elnet\n",
    "    print('Test set performance of {}: {}'.format(str(elnet).split('(')[0],elnet_score))\n",
    "except:\n",
    "    pass   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best performing model was Lasso,\n",
      "with test set performance of 0.9221\n"
     ]
    }
   ],
   "source": [
    "high_score = max(models.keys())\n",
    "print('Best performing model was {},\\nwith test set performance of {}'.format(\n",
    "    str(models[high_score]).split('(')[0], round(high_score,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T15:22:46.076866Z",
     "start_time": "2018-05-17T15:22:46.072853Z"
    }
   },
   "outputs": [],
   "source": [
    "# Choosing a Model and Outputting Submission:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:09.707649Z",
     "start_time": "2018-05-17T22:52:09.702291Z"
    }
   },
   "outputs": [],
   "source": [
    "# Choose a model based on test set performance:\n",
    "chosen_model = models[high_score]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-05-17T22:52:09.723381Z",
     "start_time": "2018-05-17T22:52:09.711700Z"
    }
   },
   "outputs": [],
   "source": [
    "if submission_path:\n",
    "\n",
    "    kaggl_preds = chosen_model.predict(kaggl_reduced)\n",
    "\n",
    "    kaggl_id = pd.read_csv('data/test.csv')['Id']\n",
    "\n",
    "    sample_submission = pd.read_csv('data/sample_submission.csv')\n",
    "    submission_columns= sample_submission.columns\n",
    "\n",
    "    submission = pd.DataFrame({submission_columns[0]:kaggl_id,\n",
    "                               submission_columns[1]:kaggl_preds})\n",
    "\n",
    "    submission.to_csv(submission_path, index=False) "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:anaconda3]",
   "language": "python",
   "name": "conda-env-anaconda3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}