1 - Data Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": [
     "settings"
    ]
   },
   "outputs": [],
   "source": [
    "import matplotlib.style as style\n",
    "import pandas as pd\n",
    "style.use('seaborn-whitegrid')\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exception reporting mode: Plain\n"
     ]
    }
   ],
   "source": [
    "# Change format of charts to .svg\n",
    "% config InlineBackend.figure_format = 'svg'\n",
    "\n",
    "%xmode Plain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('telco_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7043 entries, 0 to 7042\n",
      "Data columns (total 21 columns):\n",
      "customerID          7043 non-null object\n",
      "gender              7043 non-null object\n",
      "SeniorCitizen       7043 non-null int64\n",
      "Partner             7043 non-null object\n",
      "Dependents          7043 non-null object\n",
      "tenure              7043 non-null int64\n",
      "PhoneService        7043 non-null object\n",
      "MultipleLines       7043 non-null object\n",
      "InternetService     7043 non-null object\n",
      "OnlineSecurity      7043 non-null object\n",
      "OnlineBackup        7043 non-null object\n",
      "DeviceProtection    7043 non-null object\n",
      "TechSupport         7043 non-null object\n",
      "StreamingTV         7043 non-null object\n",
      "StreamingMovies     7043 non-null object\n",
      "Contract            7043 non-null object\n",
      "PaperlessBilling    7043 non-null object\n",
      "PaymentMethod       7043 non-null object\n",
      "MonthlyCharges      7043 non-null float64\n",
      "TotalCharges        7043 non-null object\n",
      "Churn               7043 non-null object\n",
      "dtypes: float64(1), int64(2), object(18)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7043"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for unique record ID #s - all were unique\n",
    "df['customerID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop this column since we no longer need it\n",
    "df.drop('customerID', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "copies = df.duplicated(subset=['gender', 'SeniorCitizen', 'Partner', 'Dependents',\n",
    "                               'tenure', 'PhoneService', 'MultipleLines', 'InternetService',\n",
    "                               'OnlineSecurity', 'OnlineBackup', 'DeviceProtection', 'TechSupport',\n",
    "                               'StreamingTV', 'StreamingMovies', 'Contract', 'PaperlessBilling',\n",
    "                               'PaymentMethod', 'MonthlyCharges', 'TotalCharges', 'Churn'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    7021\n",
       "True     22  \n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "copies.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I subsequently explored those 22 'duplicates' but determined they were all due to having the same characteristics -\n",
    "people in their first month of payments who had the same services and had paid the same amount."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['gender', 'SeniorCitizen', 'Partner', 'Dependents', 'tenure',\n",
       "       'PhoneService', 'MultipleLines', 'InternetService', 'OnlineSecurity',\n",
       "       'OnlineBackup', 'DeviceProtection', 'TechSupport', 'StreamingTV',\n",
       "       'StreamingMovies', 'Contract', 'PaperlessBilling', 'PaymentMethod',\n",
       "       'MonthlyCharges', 'TotalCharges', 'Churn'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Female</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>1</td>\n",
       "      <td>No</td>\n",
       "      <td>No phone service</td>\n",
       "      <td>DSL</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>Yes</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Male</td>\n",
       "      <td>0</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>34</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>DSL</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>Yes</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>No</td>\n",
       "      <td>One year</td>\n",
       "      <td>No</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender  SeniorCitizen Partner Dependents  tenure PhoneService  \\\n",
       "0  Female  0              Yes     No         1       No            \n",
       "1  Male    0              No      No         34      Yes           \n",
       "\n",
       "      MultipleLines InternetService OnlineSecurity OnlineBackup  \\\n",
       "0  No phone service  DSL             No             Yes           \n",
       "1  No                DSL             Yes            No            \n",
       "\n",
       "  DeviceProtection TechSupport StreamingTV StreamingMovies        Contract  \\\n",
       "0  No               No          No          No              Month-to-month   \n",
       "1  Yes              No          No          No              One year         \n",
       "\n",
       "  PaperlessBilling     PaymentMethod  MonthlyCharges TotalCharges Churn  \n",
       "0  Yes              Electronic check  29.85           29.85        No    \n",
       "1  No               Mailed check      56.95           1889.5       No    "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at data to see if we need to recode any variables\n",
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Coded binary categories as 0 and 1 in preparation for regression\n",
    "binary_dict = {'No': 0, 'No internet service': 0, 'No phone service': 0, 'Yes': 1}\n",
    "\n",
    "def convert_cols_to_binary(col_names: list):\n",
    "    for col in col_names:\n",
    "        df[col] = df[col].map(binary_dict)\n",
    "\n",
    "\n",
    "cols = ['Partner', 'Dependents', 'PhoneService', 'MultipleLines', 'OnlineSecurity', 'OnlineBackup', 'DeviceProtection',\n",
    "        'TechSupport', 'StreamingTV', 'StreamingMovies', 'PaperlessBilling', 'Churn']\n",
    "\n",
    "convert_cols_to_binary(cols)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>female</th>\n",
       "      <th>SeniorCitizen</th>\n",
       "      <th>Partner</th>\n",
       "      <th>Dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>PhoneService</th>\n",
       "      <th>MultipleLines</th>\n",
       "      <th>InternetService</th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "      <th>Contract</th>\n",
       "      <th>PaperlessBilling</th>\n",
       "      <th>PaymentMethod</th>\n",
       "      <th>MonthlyCharges</th>\n",
       "      <th>TotalCharges</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7038</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>One year</td>\n",
       "      <td>1</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>84.80</td>\n",
       "      <td>1990.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7039</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>72</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>One year</td>\n",
       "      <td>1</td>\n",
       "      <td>Credit card (automatic)</td>\n",
       "      <td>103.20</td>\n",
       "      <td>7362.9</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7040</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.60</td>\n",
       "      <td>346.45</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7041</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>74.40</td>\n",
       "      <td>306.6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7042</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>66</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Two year</td>\n",
       "      <td>1</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>105.65</td>\n",
       "      <td>6844.5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7043 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      female  SeniorCitizen  Partner  Dependents  tenure  PhoneService  \\\n",
       "0     1       0              1        0           1       0              \n",
       "1     0       0              0        0           34      1              \n",
       "2     0       0              0        0           2       1              \n",
       "3     0       0              0        0           45      0              \n",
       "4     1       0              0        0           2       1              \n",
       "...  ..      ..             ..       ..          ..      ..              \n",
       "7038  0       0              1        1           24      1              \n",
       "7039  1       0              1        1           72      1              \n",
       "7040  1       0              1        1           11      0              \n",
       "7041  0       1              1        0           4       1              \n",
       "7042  0       0              0        0           66      1              \n",
       "\n",
       "      MultipleLines InternetService  OnlineSecurity  OnlineBackup  \\\n",
       "0     0              DSL             0               1              \n",
       "1     0              DSL             1               0              \n",
       "2     0              DSL             1               1              \n",
       "3     0              DSL             1               0              \n",
       "4     0              Fiber optic     0               0              \n",
       "...  ..                      ...    ..              ..              \n",
       "7038  1              DSL             1               0              \n",
       "7039  1              Fiber optic     0               1              \n",
       "7040  0              DSL             1               0              \n",
       "7041  1              Fiber optic     0               0              \n",
       "7042  0              Fiber optic     1               0              \n",
       "\n",
       "      DeviceProtection  TechSupport  StreamingTV  StreamingMovies  \\\n",
       "0     0                 0            0            0                 \n",
       "1     1                 0            0            0                 \n",
       "2     0                 0            0            0                 \n",
       "3     1                 1            0            0                 \n",
       "4     0                 0            0            0                 \n",
       "...  ..                ..           ..           ..                 \n",
       "7038  1                 1            1            1                 \n",
       "7039  1                 0            1            1                 \n",
       "7040  0                 0            0            0                 \n",
       "7041  0                 0            0            0                 \n",
       "7042  1                 1            1            1                 \n",
       "\n",
       "            Contract  PaperlessBilling              PaymentMethod  \\\n",
       "0     Month-to-month  1                 Electronic check            \n",
       "1     One year        0                 Mailed check                \n",
       "2     Month-to-month  1                 Mailed check                \n",
       "3     One year        0                 Bank transfer (automatic)   \n",
       "4     Month-to-month  1                 Electronic check            \n",
       "...              ... ..                              ...            \n",
       "7038  One year        1                 Mailed check                \n",
       "7039  One year        1                 Credit card (automatic)     \n",
       "7040  Month-to-month  1                 Electronic check            \n",
       "7041  Month-to-month  1                 Mailed check                \n",
       "7042  Two year        1                 Bank transfer (automatic)   \n",
       "\n",
       "      MonthlyCharges TotalCharges  Churn  \n",
       "0     29.85           29.85        0      \n",
       "1     56.95           1889.5       0      \n",
       "2     53.85           108.15       1      \n",
       "3     42.30           1840.75      0      \n",
       "4     70.70           151.65       1      \n",
       "...     ...              ...      ..      \n",
       "7038  84.80           1990.5       0      \n",
       "7039  103.20          7362.9       0      \n",
       "7040  29.60           346.45       0      \n",
       "7041  74.40           306.6        1      \n",
       "7042  105.65          6844.5       0      \n",
       "\n",
       "[7043 rows x 20 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Code gender as binary\n",
    "gender_dict = {'Male': 0, 'Female': 1}\n",
    "df['gender'] = df['gender'].map(gender_dict)\n",
    "\n",
    "# Rename column to 'female' since 1's now represent females\n",
    "df.rename(columns={'gender': 'female'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['gender', 'SeniorCitizen', 'Partner', 'Dependents', 'tenure',\n",
       "       'PhoneService', 'MultipleLines', 'InternetService', 'OnlineSecurity',\n",
       "       'OnlineBackup', 'DeviceProtection', 'TechSupport', 'StreamingTV',\n",
       "       'StreamingMovies', 'Contract', 'PaperlessBilling', 'PaymentMethod',\n",
       "       'MonthlyCharges', 'TotalCharges', 'Churn'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confirm that our column renamed correctly - it did\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OnlineSecurity</th>\n",
       "      <th>OnlineBackup</th>\n",
       "      <th>DeviceProtection</th>\n",
       "      <th>TechSupport</th>\n",
       "      <th>StreamingTV</th>\n",
       "      <th>StreamingMovies</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7038</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7039</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7040</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7041</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7042</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7043 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      OnlineSecurity  OnlineBackup  DeviceProtection  TechSupport  \\\n",
       "0     0               1             0                 0             \n",
       "1     1               0             1                 0             \n",
       "2     1               1             0                 0             \n",
       "3     1               0             1                 1             \n",
       "4     0               0             0                 0             \n",
       "...  ..              ..            ..                ..             \n",
       "7038  1               0             1                 1             \n",
       "7039  0               1             1                 0             \n",
       "7040  1               0             0                 0             \n",
       "7041  0               0             0                 0             \n",
       "7042  1               0             1                 1             \n",
       "\n",
       "      StreamingTV  StreamingMovies  \n",
       "0     0            0                \n",
       "1     0            0                \n",
       "2     0            0                \n",
       "3     0            0                \n",
       "4     0            0                \n",
       "...  ..           ..                \n",
       "7038  1            1                \n",
       "7039  1            1                \n",
       "7040  0            0                \n",
       "7041  0            0                \n",
       "7042  1            1                \n",
       "\n",
       "[7043 rows x 6 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recast binary columns as integers, as some of them were floats\n",
    "df[['OnlineSecurity', 'OnlineBackup', 'DeviceProtection', 'TechSupport',\n",
    "    'StreamingTV', 'StreamingMovies']].astype('int', inplace=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tried recasting TotalCharges to float to divide it by tenure, but got an error as there are 11 values in the column showing as ' '. This occurred because some people had not had any charges yet. I then dropped those records (only 11 of them) in order to allow us to cast the column as float."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dropping the 11 ' ' values from TotalCharges column\n",
    "df = df[df['TotalCharges']!=' ']\n",
    "\n",
    "# Then recast the column as float\n",
    "df['TotalCharges'] = df['TotalCharges'].astype('float')\n",
    "\n",
    "# Created new feature column to measure average monthly charges\n",
    "df['AvgMonthlyCharges'] = df['TotalCharges']/df['tenure']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create dummy variables in preparation for regression.\n",
    "# Drop first binary column for dummy variables to prevent redundant columns.\n",
    "test = pd.get_dummies(data=df, columns=['InternetService', 'Contract', 'PaymentMethod'], drop_first=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change name of columnns for both dataframes to make them more easier to read and work with\n",
    "df.columns = ['gender', 'senior', 'partner', 'dependents', 'tenure',\n",
    "              'phone_service', 'multiple_lines', 'internet_service', 'online_security', 'online_backup',\n",
    "              'device_protection', 'tech_support', 'streaming_tv', 'streaming_movies',\n",
    "              'contract', 'paperless_billing', 'payment_method', 'monthly_charges', 'total_charges', 'churn',\n",
    "              'avg_monthly_charges']\n",
    "\n",
    "test.columns = ['gender', 'senior', 'partner', 'dependents', 'tenure',\n",
    "                'phone_service', 'multiple_lines', 'online_security', 'online_backup',\n",
    "                'device_protection', 'tech_support', 'streaming_tv', 'streaming_movies',\n",
    "                'paperless_billing', 'monthly_charges', 'total_charges', 'churn',\n",
    "                'avg_monthly_charges', 'internet_service-fiber_optic',\n",
    "                'internet_service-no', 'contract-one_year', 'contract-two_year',\n",
    "                'payment_method-credit_card_auto',\n",
    "                'payment_method-electronic_check', 'payment_method-mailed_check']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Saving clean data and clean data + dummy variables for regression. Commenting code out below so we don't overwrite our current data files unless we mean to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Save output as CSV\n",
    "# df.to_csv('data/clean_data.csv', index=False)\n",
    "# test.to_csv('data/clean_data_encoded_for_regression.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/clean_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gender</th>\n",
       "      <th>senior</th>\n",
       "      <th>partner</th>\n",
       "      <th>dependents</th>\n",
       "      <th>tenure</th>\n",
       "      <th>phone_service</th>\n",
       "      <th>multiple_lines</th>\n",
       "      <th>internet_service</th>\n",
       "      <th>online_security</th>\n",
       "      <th>online_backup</th>\n",
       "      <th>device_protection</th>\n",
       "      <th>tech_support</th>\n",
       "      <th>streaming_tv</th>\n",
       "      <th>streaming_movies</th>\n",
       "      <th>contract</th>\n",
       "      <th>paperless_billing</th>\n",
       "      <th>payment_method</th>\n",
       "      <th>monthly_charges</th>\n",
       "      <th>total_charges</th>\n",
       "      <th>churn</th>\n",
       "      <th>avg_monthly_charges</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>29.85</td>\n",
       "      <td>29.85</td>\n",
       "      <td>0</td>\n",
       "      <td>29.850000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>56.95</td>\n",
       "      <td>1889.50</td>\n",
       "      <td>0</td>\n",
       "      <td>55.573529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Mailed check</td>\n",
       "      <td>53.85</td>\n",
       "      <td>108.15</td>\n",
       "      <td>1</td>\n",
       "      <td>54.075000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>45</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>DSL</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>One year</td>\n",
       "      <td>0</td>\n",
       "      <td>Bank transfer (automatic)</td>\n",
       "      <td>42.30</td>\n",
       "      <td>1840.75</td>\n",
       "      <td>0</td>\n",
       "      <td>40.905556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Fiber optic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Month-to-month</td>\n",
       "      <td>1</td>\n",
       "      <td>Electronic check</td>\n",
       "      <td>70.70</td>\n",
       "      <td>151.65</td>\n",
       "      <td>1</td>\n",
       "      <td>75.825000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   gender  senior  partner  dependents  tenure  phone_service  multiple_lines  \\\n",
       "0  1       0       1        0           1       0              0                \n",
       "1  0       0       0        0           34      1              0                \n",
       "2  0       0       0        0           2       1              0                \n",
       "3  0       0       0        0           45      0              0                \n",
       "4  1       0       0        0           2       1              0                \n",
       "\n",
       "  internet_service  online_security  online_backup  device_protection  \\\n",
       "0  DSL              0                1              0                   \n",
       "1  DSL              1                0              1                   \n",
       "2  DSL              1                1              0                   \n",
       "3  DSL              1                0              1                   \n",
       "4  Fiber optic      0                0              0                   \n",
       "\n",
       "   tech_support  streaming_tv  streaming_movies        contract  \\\n",
       "0  0             0             0                 Month-to-month   \n",
       "1  0             0             0                 One year         \n",
       "2  0             0             0                 Month-to-month   \n",
       "3  1             0             0                 One year         \n",
       "4  0             0             0                 Month-to-month   \n",
       "\n",
       "   paperless_billing             payment_method  monthly_charges  \\\n",
       "0  1                  Electronic check           29.85             \n",
       "1  0                  Mailed check               56.95             \n",
       "2  1                  Mailed check               53.85             \n",
       "3  0                  Bank transfer (automatic)  42.30             \n",
       "4  1                  Electronic check           70.70             \n",
       "\n",
       "   total_charges  churn  avg_monthly_charges  \n",
       "0  29.85          0      29.850000            \n",
       "1  1889.50        0      55.573529            \n",
       "2  108.15         1      54.075000            \n",
       "3  1840.75        0      40.905556            \n",
       "4  151.65         1      75.825000            "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7032.000000\n",
       "mean     64.798208  \n",
       "std      30.085974  \n",
       "min      18.250000  \n",
       "25%      35.587500  \n",
       "50%      70.350000  \n",
       "75%      89.862500  \n",
       "max      118.750000 \n",
       "Name: monthly_charges, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['monthly_charges'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7032.000000\n",
       "mean     32.421786  \n",
       "std      24.545260  \n",
       "min      1.000000   \n",
       "25%      9.000000   \n",
       "50%      29.000000  \n",
       "75%      55.000000  \n",
       "max      72.000000  \n",
       "Name: tenure, dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['tenure'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Fiber optic    3096\n",
       "DSL            2416\n",
       "No             1520\n",
       "Name: internet_service, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['internet_service'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "internet_service\n",
       "DSL            58.088017\n",
       "Fiber optic    91.500129\n",
       "No             21.076283\n",
       "Name: monthly_charges, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(df['internet_service'])['monthly_charges'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = df.groupby(df['tenure'])['monthly_charges'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "phone_service\n",
       "0    41.992500\n",
       "1    67.239625\n",
       "Name: monthly_charges, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(df['phone_service'])['monthly_charges'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['gender', 'senior', 'partner', 'dependents', 'tenure', 'phone_service',\n",
       "       'multiple_lines', 'internet_service', 'online_security',\n",
       "       'online_backup', 'device_protection', 'tech_support', 'streaming_tv',\n",
       "       'streaming_movies', 'contract', 'paperless_billing', 'payment_method',\n",
       "       'monthly_charges', 'total_charges', 'churn', 'avg_monthly_charges'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "EOL while scanning string literal (<ipython-input-28-04d0430dc238>, line 4)",
     "traceback": [
      "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-28-04d0430dc238>\"\u001b[0;36m, line \u001b[0;32m4\u001b[0m\n\u001b[0;31m    'internet_service-no]]\u001b[0m\n\u001b[0m                          ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m EOL while scanning string literal\n"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "X = df[['phone_service',\n",
    "        'multiple_lines', 'online_security', 'online_backup',\n",
    "        'device_protection', 'tech_support', 'streaming_tv', 'streaming_movies', 'internet_service-fiber_optic',\n",
    "        'internet_service-no']]\n",
    "\n",
    "y = df['monthly_charges']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc-autonumbering": false,
  "toc-showcode": true,
  "toc-showmarkdowntxt": false,
  "toc-showtags": true
 },
 "nbformat": 4,
 "nbformat_minor": 2
}