models/5_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<pre>\n",
    "Reference: \n",
    "Machine Learning- Sudeshna Sarkar (Logistic Regression): https://www.youtube.com/watch?v=CE03E80wbRE\n",
    "</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We will build a basic logistic regression model on titanic dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_curve, auc, roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('../data/titanic_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df.replace([np.inf, -np.inf], np.nan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['Age'].fillna(df['Age'].mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['has_cabin'] = df['Cabin'].apply(lambda val: 0 if pd.isnull(val) else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PassengerId', 'Survived', 'Pclass', 'Name', 'Sex', 'Age', 'SibSp',\n",
       "       'Parch', 'Ticket', 'Fare', 'Cabin', 'Embarked', 'has_cabin'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = df[['Age', 'Survived', 'Sex', 'PassengerId', 'has_cabin']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.set_index('PassengerId', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df['Sex'] = df['Sex'].map({'male':0, 'female':1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 4)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Sex</th>\n",
       "      <th>has_cabin</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PassengerId</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>22.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>26.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Age  Survived  Sex  has_cabin\n",
       "PassengerId                                \n",
       "1            22.0         0    0          0\n",
       "2            38.0         1    1          1\n",
       "3            26.0         1    1          0\n",
       "4            35.0         1    1          1\n",
       "5            35.0         0    0          0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "min_max_scaler = preprocessing.MinMaxScaler(feature_range=(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_survived = df.pop('Survived')\n",
    "min_max_scaler.fit(df)\n",
    "df['Survived'] = df_survived"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test = train_test_split(df, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train = X_train.pop('Survived')\n",
    "Y_test = X_test.pop('Survived')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = min_max_scaler.transform(X_train)\n",
    "X_test = min_max_scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a custom logistic regression model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sigmoid is simply: 1/(1+ e^(-x))\n",
    "def sigmoid(values):\n",
    "    return 1/(1+np.exp(-values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.73105858, 0.88079708, 0.95257413])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(np.array([1,2,3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# log_likelihood which we want to maximise\n",
    "# Formula:\n",
    "# Sigma (y_i * log(h(x_i)) + (1-y_i) log(1-h(x_i)))\n",
    "# where y_i is ith output value\n",
    "# h(x_i) is sigmoid(B'X) [sigmoid of output of i_th input features]\n",
    "def log_likelihood(expected_output, predicted_output):\n",
    "    return np.sum(expected_output * np.log(predicted_output) + (1 - expected_output) * np.log(1 - predicted_output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "log_likelihood 1 -0.7905395265685948\n",
      "log_likelihood 2  -0.6727564909122112\n"
     ]
    }
   ],
   "source": [
    "print('log_likelihood 1', log_likelihood(np.array([1,1,0,0]), np.array([0.9,0.8,0.3,0.1])))\n",
    "print('log_likelihood 2 ', log_likelihood(np.array([1,1,0,0]), np.array([0.9,0.9,0.3,0.1])))\n",
    "# log_likelihood 2 > 1 because the predicted values are closer to expected values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# if we wan't an intercept value then we add a new feature as all 1s\n",
    "# Formula: for each step\n",
    "# compute the prediction using B' * X done where B is weights and X is features\n",
    "# prediccted output is passed to sigmoid fuction\n",
    "# difference in prediction and actual is used to get error\n",
    "# this error is used to get get gradient along with learning rate which is used to update weight\n",
    "\n",
    "def logistic_regression(features, target, num_steps, learning_rate, fit_intercept=True, log_steps=100):\n",
    "    if fit_intercept:\n",
    "        # adding intercept, Beta_0\n",
    "        intercept = np.ones((features.shape[0], 1))\n",
    "        features = np.hstack((intercept, features))\n",
    "    \n",
    "    weights = np.zeros(features.shape[1])\n",
    "    for i in range(num_steps):\n",
    "        prediction = np.dot(features, weights)\n",
    "        predicted_output = sigmoid(prediction)\n",
    "        \n",
    "        # Gradient descent to update weights\n",
    "        output_error = target - predicted_output\n",
    "        gradient = np.dot(features.T, output_error)\n",
    "        weights += learning_rate * gradient\n",
    "        \n",
    "        if i % log_steps == 0:\n",
    "            print('step:', i, 'log_likelihood:', log_likelihood(target, predicted_output))\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Same process as above but no learning rate or updateing weights. just making predictions.\n",
    "def logistic_predict(features, weights, fit_intercept=True):\n",
    "    if fit_intercept:\n",
    "        # adding intercept, Beta_0\n",
    "        intercept = np.ones((features.shape[0], 1))\n",
    "        features = np.hstack((intercept, features))\n",
    "    prediction = np.dot(features, weights)\n",
    "    predicted_output = sigmoid(prediction)\n",
    "    return predicted_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 0 log_likelihood: -431.8306934888469\n",
      "step: 1000 log_likelihood: -297.7205450866993\n",
      "step: 2000 log_likelihood: -296.91989396539253\n",
      "step: 3000 log_likelihood: -296.65668398443137\n",
      "step: 4000 log_likelihood: -296.5500469031612\n",
      "step: 5000 log_likelihood: -296.50623906742\n",
      "step: 6000 log_likelihood: -296.4881532872569\n",
      "step: 7000 log_likelihood: -296.48066315235764\n",
      "step: 8000 log_likelihood: -296.4775547404274\n",
      "step: 9000 log_likelihood: -296.4762630089804\n"
     ]
    }
   ],
   "source": [
    "weights = logistic_regression(X_train, Y_train, num_steps=10000, learning_rate=5e-5, log_steps=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intercept -0.025307611845771318 coef_ [-0.77629612  1.22603635  0.85276009]\n"
     ]
    }
   ],
   "source": [
    "print('intercept', weights[0], 'coef_', weights[1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8478223446376313"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(Y_test, logistic_predict(X_test, weights))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build a logistic regression model with Sklean for validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibLinear]"
     ]
    }
   ],
   "source": [
    "# C = 5 to reduce regularization on coeficients, as we don't have any regularization in our model.\n",
    "model = LogisticRegression(fit_intercept=True, C=5, max_iter=10000, verbose=5)\n",
    "model = model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8478223446376313"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(Y_test, model.predict_proba(X_test)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intercept [-0.02545202] coef [[-0.76697725  1.22293027  0.84869292]]\n"
     ]
    }
   ],
   "source": [
    "print('intercept', model.intercept_, 'coef', model.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}