DayByDay.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(style=\"white\", color_codes=True)\n",
    "sns.set_context(rc={\"font.family\":'sans',\"font.size\":24,\"axes.titlesize\":24,\"axes.labelsize\":24})  \n",
    "\n",
    "import warnings \n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./dataset/enriched_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     523000\n",
       "unique       749\n",
       "top         1748\n",
       "freq         729\n",
       "Name: StoreID, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_cat=df['StoreID']\n",
    "dataset_cat.astype('object').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>StoreID</th>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <td>2016-03-23</td>\n",
       "      <td>2016-03-24</td>\n",
       "      <td>2016-03-25</td>\n",
       "      <td>2016-03-26</td>\n",
       "      <td>2016-03-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsOpen</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HasPromotions</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType</th>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AssortmentType</th>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NearestCompetitor</th>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NumberOfCustomers</th>\n",
       "      <td>541</td>\n",
       "      <td>626</td>\n",
       "      <td>625</td>\n",
       "      <td>511</td>\n",
       "      <td>622.769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NumberOfSales</th>\n",
       "      <td>7109</td>\n",
       "      <td>8041</td>\n",
       "      <td>7890</td>\n",
       "      <td>6005</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_AreaKM2</th>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_GDP</th>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_PopulationK</th>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CloudCover</th>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Events</th>\n",
       "      <td>Rain</td>\n",
       "      <td>Rain-Snow</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Dew_PointC</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-5</td>\n",
       "      <td>-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Humidity</th>\n",
       "      <td>100</td>\n",
       "      <td>87</td>\n",
       "      <td>93</td>\n",
       "      <td>64</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Sea_Level_PressurehPa</th>\n",
       "      <td>1012</td>\n",
       "      <td>1020</td>\n",
       "      <td>1020</td>\n",
       "      <td>1016</td>\n",
       "      <td>1016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_TemperatureC</th>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_VisibilityKm</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Wind_SpeedKm_h</th>\n",
       "      <td>32</td>\n",
       "      <td>21</td>\n",
       "      <td>34</td>\n",
       "      <td>34</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Dew_PointC</th>\n",
       "      <td>3</td>\n",
       "      <td>-2</td>\n",
       "      <td>-3</td>\n",
       "      <td>-7</td>\n",
       "      <td>-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Humidity</th>\n",
       "      <td>89</td>\n",
       "      <td>71</td>\n",
       "      <td>61</td>\n",
       "      <td>52</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Sea_Level_PressurehPa</th>\n",
       "      <td>1004</td>\n",
       "      <td>1017</td>\n",
       "      <td>1016</td>\n",
       "      <td>1015</td>\n",
       "      <td>1013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_TemperatureC</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_VisibilityKm</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Wind_SpeedKm_h</th>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>14</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Dew_PointC</th>\n",
       "      <td>-1</td>\n",
       "      <td>-4</td>\n",
       "      <td>-6</td>\n",
       "      <td>-10</td>\n",
       "      <td>-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Humidity</th>\n",
       "      <td>58</td>\n",
       "      <td>51</td>\n",
       "      <td>31</td>\n",
       "      <td>34</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Sea_Level_PressurehPa</th>\n",
       "      <td>1001</td>\n",
       "      <td>1012</td>\n",
       "      <td>1014</td>\n",
       "      <td>1014</td>\n",
       "      <td>1011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_TemperatureC</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_VisibilitykM</th>\n",
       "      <td>21</td>\n",
       "      <td>22</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precipitationmm</th>\n",
       "      <td>5.08</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindDirDegrees</th>\n",
       "      <td>191</td>\n",
       "      <td>331</td>\n",
       "      <td>59</td>\n",
       "      <td>67</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Day</th>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>26</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfYear</th>\n",
       "      <td>83</td>\n",
       "      <td>84</td>\n",
       "      <td>85</td>\n",
       "      <td>86</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Month</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Year</th>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-14</th>\n",
       "      <td>8804</td>\n",
       "      <td>7823</td>\n",
       "      <td>7989</td>\n",
       "      <td>5895</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-7</th>\n",
       "      <td>6014</td>\n",
       "      <td>6087</td>\n",
       "      <td>6962</td>\n",
       "      <td>6618</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-1</th>\n",
       "      <td>8155</td>\n",
       "      <td>7109</td>\n",
       "      <td>8041</td>\n",
       "      <td>7890</td>\n",
       "      <td>6005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-2</th>\n",
       "      <td>9183</td>\n",
       "      <td>8155</td>\n",
       "      <td>7109</td>\n",
       "      <td>8041</td>\n",
       "      <td>7890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-9</th>\n",
       "      <td>6348</td>\n",
       "      <td>4738</td>\n",
       "      <td>6014</td>\n",
       "      <td>6087</td>\n",
       "      <td>6962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-8</th>\n",
       "      <td>4738</td>\n",
       "      <td>6014</td>\n",
       "      <td>6087</td>\n",
       "      <td>6962</td>\n",
       "      <td>6618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-6</th>\n",
       "      <td>6087</td>\n",
       "      <td>6962</td>\n",
       "      <td>6618</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>9183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-15</th>\n",
       "      <td>9019</td>\n",
       "      <td>8804</td>\n",
       "      <td>7823</td>\n",
       "      <td>7989</td>\n",
       "      <td>5895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-21</th>\n",
       "      <td>5676</td>\n",
       "      <td>8111</td>\n",
       "      <td>8300</td>\n",
       "      <td>7154</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-13</th>\n",
       "      <td>7823</td>\n",
       "      <td>7989</td>\n",
       "      <td>5895</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>6348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-20</th>\n",
       "      <td>8111</td>\n",
       "      <td>8300</td>\n",
       "      <td>7154</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>10110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanStoreSales</th>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanRegionSales</th>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Day_of_week</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       0             1             2  \\\n",
       "StoreID                             1000          1000          1000   \n",
       "Date                          2016-03-23    2016-03-24    2016-03-25   \n",
       "IsHoliday                              0             0             0   \n",
       "IsOpen                                 1             1             1   \n",
       "HasPromotions                          1             1             1   \n",
       "StoreType                   Hyper Market  Hyper Market  Hyper Market   \n",
       "AssortmentType                   General       General       General   \n",
       "NearestCompetitor                    326           326           326   \n",
       "Region                                 7             7             7   \n",
       "NumberOfCustomers                    541           626           625   \n",
       "NumberOfSales                       7109          8041          7890   \n",
       "Region_AreaKM2                      9643          9643          9643   \n",
       "Region_GDP                         17130         17130         17130   \n",
       "Region_PopulationK                  2770          2770          2770   \n",
       "CloudCover                             6             6             5   \n",
       "Events                              Rain     Rain-Snow          None   \n",
       "Max_Dew_PointC                         6             0            -1   \n",
       "Max_Humidity                         100            87            93   \n",
       "Max_Sea_Level_PressurehPa           1012          1020          1020   \n",
       "Max_TemperatureC                       9             5             7   \n",
       "Max_VisibilityKm                      21            22            23   \n",
       "Max_Wind_SpeedKm_h                    32            21            34   \n",
       "Mean_Dew_PointC                        3            -2            -3   \n",
       "Mean_Humidity                         89            71            61   \n",
       "Mean_Sea_Level_PressurehPa          1004          1017          1016   \n",
       "Mean_TemperatureC                      5             3             3   \n",
       "Mean_VisibilityKm                     21            22            23   \n",
       "Mean_Wind_SpeedKm_h                   10            11            14   \n",
       "Min_Dew_PointC                        -1            -4            -6   \n",
       "Min_Humidity                          58            51            31   \n",
       "Min_Sea_Level_PressurehPa           1001          1012          1014   \n",
       "Min_TemperatureC                       1             1            -1   \n",
       "Min_VisibilitykM                      21            22            23   \n",
       "Precipitationmm                     5.08             0             0   \n",
       "WindDirDegrees                       191           331            59   \n",
       "D_Day                                 23            24            25   \n",
       "D_DayOfYear                           83            84            85   \n",
       "D_Month                                3             3             3   \n",
       "D_Year                              2016          2016          2016   \n",
       "t-14                                8804          7823          7989   \n",
       "t-7                                 6014          6087          6962   \n",
       "t-1                                 8155          7109          8041   \n",
       "t-2                                 9183          8155          7109   \n",
       "t-9                                 6348          4738          6014   \n",
       "t-8                                 4738          6014          6087   \n",
       "t-6                                 6087          6962          6618   \n",
       "t-15                                9019          8804          7823   \n",
       "t-21                                5676          8111          8300   \n",
       "t-13                                7823          7989          5895   \n",
       "t-20                                8111          8300          7154   \n",
       "MeanStoreSales                   7682.75       7682.75       7682.75   \n",
       "MeanRegionSales                  4698.57       4698.57       4698.57   \n",
       "D_Day_of_week                          2             3             4   \n",
       "\n",
       "                                       3             4  \n",
       "StoreID                             1000          1000  \n",
       "Date                          2016-03-26    2016-03-27  \n",
       "IsHoliday                              0             0  \n",
       "IsOpen                                 1             0  \n",
       "HasPromotions                          0             0  \n",
       "StoreType                   Hyper Market  Hyper Market  \n",
       "AssortmentType                   General       General  \n",
       "NearestCompetitor                    326           326  \n",
       "Region                                 7             7  \n",
       "NumberOfCustomers                    511       622.769  \n",
       "NumberOfSales                       6005       7675.45  \n",
       "Region_AreaKM2                      9643          9643  \n",
       "Region_GDP                         17130         17130  \n",
       "Region_PopulationK                  2770          2770  \n",
       "CloudCover                             7             8  \n",
       "Events                              None          None  \n",
       "Max_Dew_PointC                        -5           -10  \n",
       "Max_Humidity                          64            48  \n",
       "Max_Sea_Level_PressurehPa           1016          1016  \n",
       "Max_TemperatureC                       3             6  \n",
       "Max_VisibilityKm                      24            25  \n",
       "Max_Wind_SpeedKm_h                    34            35  \n",
       "Mean_Dew_PointC                       -7           -12  \n",
       "Mean_Humidity                         52            34  \n",
       "Mean_Sea_Level_PressurehPa          1015          1013  \n",
       "Mean_TemperatureC                      1             2  \n",
       "Mean_VisibilityKm                     24            25  \n",
       "Mean_Wind_SpeedKm_h                   23            24  \n",
       "Min_Dew_PointC                       -10           -14  \n",
       "Min_Humidity                          34            13  \n",
       "Min_Sea_Level_PressurehPa           1014          1011  \n",
       "Min_TemperatureC                      -1            -2  \n",
       "Min_VisibilitykM                      24            25  \n",
       "Precipitationmm                        0             0  \n",
       "WindDirDegrees                        67            59  \n",
       "D_Day                                 26            27  \n",
       "D_DayOfYear                           86            87  \n",
       "D_Month                                3             3  \n",
       "D_Year                              2016          2016  \n",
       "t-14                                5895       7675.45  \n",
       "t-7                                 6618       7675.45  \n",
       "t-1                                 7890          6005  \n",
       "t-2                                 8041          7890  \n",
       "t-9                                 6087          6962  \n",
       "t-8                                 6962          6618  \n",
       "t-6                              7675.45          9183  \n",
       "t-15                                7989          5895  \n",
       "t-21                                7154       7675.45  \n",
       "t-13                             7675.45          6348  \n",
       "t-20                             7675.45         10110  \n",
       "MeanStoreSales                   7682.75       7682.75  \n",
       "MeanRegionSales                  4698.57       4698.57  \n",
       "D_Day_of_week                          5             6  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>649</th>\n",
       "      <th>650</th>\n",
       "      <th>651</th>\n",
       "      <th>652</th>\n",
       "      <th>653</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>StoreID</th>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Date</th>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>2018-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsHoliday</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IsOpen</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HasPromotions</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>StoreType</th>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>Hyper Market</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AssortmentType</th>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "      <td>General</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NearestCompetitor</th>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region</th>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NumberOfCustomers</th>\n",
       "      <td>642</td>\n",
       "      <td>746</td>\n",
       "      <td>333</td>\n",
       "      <td>622.769</td>\n",
       "      <td>536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NumberOfSales</th>\n",
       "      <td>8540</td>\n",
       "      <td>10364</td>\n",
       "      <td>4676</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>6267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_AreaKM2</th>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "      <td>9643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_GDP</th>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "      <td>17130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Region_PopulationK</th>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "      <td>2770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CloudCover</th>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Events</th>\n",
       "      <td>Snow</td>\n",
       "      <td>Rain-Snow</td>\n",
       "      <td>Rain-Snow</td>\n",
       "      <td>Fog</td>\n",
       "      <td>Rain</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Dew_PointC</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Humidity</th>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Sea_Level_PressurehPa</th>\n",
       "      <td>1037</td>\n",
       "      <td>1038</td>\n",
       "      <td>1037</td>\n",
       "      <td>1038</td>\n",
       "      <td>1035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_TemperatureC</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_VisibilityKm</th>\n",
       "      <td>670</td>\n",
       "      <td>671</td>\n",
       "      <td>672</td>\n",
       "      <td>673</td>\n",
       "      <td>674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Max_Wind_SpeedKm_h</th>\n",
       "      <td>26</td>\n",
       "      <td>19</td>\n",
       "      <td>14</td>\n",
       "      <td>19</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Dew_PointC</th>\n",
       "      <td>-4</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Humidity</th>\n",
       "      <td>85</td>\n",
       "      <td>89</td>\n",
       "      <td>94</td>\n",
       "      <td>96</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Sea_Level_PressurehPa</th>\n",
       "      <td>1032</td>\n",
       "      <td>1035</td>\n",
       "      <td>1035</td>\n",
       "      <td>1036</td>\n",
       "      <td>1030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_TemperatureC</th>\n",
       "      <td>-3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_VisibilityKm</th>\n",
       "      <td>670</td>\n",
       "      <td>671</td>\n",
       "      <td>672</td>\n",
       "      <td>673</td>\n",
       "      <td>674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mean_Wind_SpeedKm_h</th>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Dew_PointC</th>\n",
       "      <td>-8</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Humidity</th>\n",
       "      <td>64</td>\n",
       "      <td>71</td>\n",
       "      <td>84</td>\n",
       "      <td>87</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_Sea_Level_PressurehPa</th>\n",
       "      <td>1029</td>\n",
       "      <td>1033</td>\n",
       "      <td>1033</td>\n",
       "      <td>1033</td>\n",
       "      <td>1028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_TemperatureC</th>\n",
       "      <td>-8</td>\n",
       "      <td>-4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Min_VisibilitykM</th>\n",
       "      <td>670</td>\n",
       "      <td>671</td>\n",
       "      <td>672</td>\n",
       "      <td>673</td>\n",
       "      <td>674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precipitationmm</th>\n",
       "      <td>0.25</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WindDirDegrees</th>\n",
       "      <td>226</td>\n",
       "      <td>234</td>\n",
       "      <td>195</td>\n",
       "      <td>205</td>\n",
       "      <td>206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Day</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_DayOfYear</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Month</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Year</th>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-14</th>\n",
       "      <td>12353</td>\n",
       "      <td>10520</td>\n",
       "      <td>9479</td>\n",
       "      <td>10529</td>\n",
       "      <td>9050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-7</th>\n",
       "      <td>11881</td>\n",
       "      <td>10650</td>\n",
       "      <td>4498</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-1</th>\n",
       "      <td>7675.45</td>\n",
       "      <td>8540</td>\n",
       "      <td>10364</td>\n",
       "      <td>4676</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-2</th>\n",
       "      <td>8930</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>8540</td>\n",
       "      <td>10364</td>\n",
       "      <td>4676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-9</th>\n",
       "      <td>9835</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>11881</td>\n",
       "      <td>10650</td>\n",
       "      <td>4498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-8</th>\n",
       "      <td>7675.45</td>\n",
       "      <td>11881</td>\n",
       "      <td>10650</td>\n",
       "      <td>4498</td>\n",
       "      <td>7675.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-6</th>\n",
       "      <td>10650</td>\n",
       "      <td>4498</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>8930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-15</th>\n",
       "      <td>7675.45</td>\n",
       "      <td>12353</td>\n",
       "      <td>10520</td>\n",
       "      <td>9479</td>\n",
       "      <td>10529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-21</th>\n",
       "      <td>7368</td>\n",
       "      <td>6858</td>\n",
       "      <td>5890</td>\n",
       "      <td>6913</td>\n",
       "      <td>7282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-13</th>\n",
       "      <td>10520</td>\n",
       "      <td>9479</td>\n",
       "      <td>10529</td>\n",
       "      <td>9050</td>\n",
       "      <td>9835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>t-20</th>\n",
       "      <td>6858</td>\n",
       "      <td>5890</td>\n",
       "      <td>6913</td>\n",
       "      <td>7282</td>\n",
       "      <td>9179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanStoreSales</th>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "      <td>7682.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MeanRegionSales</th>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "      <td>4698.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D_Day_of_week</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>_NumberOfSales</th>\n",
       "      <td>8540</td>\n",
       "      <td>10364</td>\n",
       "      <td>4676</td>\n",
       "      <td>7675.45</td>\n",
       "      <td>6267</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     649           650           651  \\\n",
       "StoreID                             1000          1000          1000   \n",
       "Date                          2018-01-01    2018-01-02    2018-01-03   \n",
       "IsHoliday                              0             0             0   \n",
       "IsOpen                                 1             1             1   \n",
       "HasPromotions                          0             0             0   \n",
       "StoreType                   Hyper Market  Hyper Market  Hyper Market   \n",
       "AssortmentType                   General       General       General   \n",
       "NearestCompetitor                    326           326           326   \n",
       "Region                                 7             7             7   \n",
       "NumberOfCustomers                    642           746           333   \n",
       "NumberOfSales                       8540         10364          4676   \n",
       "Region_AreaKM2                      9643          9643          9643   \n",
       "Region_GDP                         17130         17130         17130   \n",
       "Region_PopulationK                  2770          2770          2770   \n",
       "CloudCover                             6             7             7   \n",
       "Events                              Snow     Rain-Snow     Rain-Snow   \n",
       "Max_Dew_PointC                         0             1             2   \n",
       "Max_Humidity                         100           100           100   \n",
       "Max_Sea_Level_PressurehPa           1037          1038          1037   \n",
       "Max_TemperatureC                       2             4             2   \n",
       "Max_VisibilityKm                     670           671           672   \n",
       "Max_Wind_SpeedKm_h                    26            19            14   \n",
       "Mean_Dew_PointC                       -4            -1             1   \n",
       "Mean_Humidity                         85            89            94   \n",
       "Mean_Sea_Level_PressurehPa          1032          1035          1035   \n",
       "Mean_TemperatureC                     -3             0             1   \n",
       "Mean_VisibilityKm                    670           671           672   \n",
       "Mean_Wind_SpeedKm_h                   11            10            10   \n",
       "Min_Dew_PointC                        -8            -4             0   \n",
       "Min_Humidity                          64            71            84   \n",
       "Min_Sea_Level_PressurehPa           1029          1033          1033   \n",
       "Min_TemperatureC                      -8            -4             0   \n",
       "Min_VisibilitykM                     670           671           672   \n",
       "Precipitationmm                     0.25             0             0   \n",
       "WindDirDegrees                       226           234           195   \n",
       "D_Day                                  1             2             3   \n",
       "D_DayOfYear                            1             2             3   \n",
       "D_Month                                1             1             1   \n",
       "D_Year                              2018          2018          2018   \n",
       "t-14                               12353         10520          9479   \n",
       "t-7                                11881         10650          4498   \n",
       "t-1                              7675.45          8540         10364   \n",
       "t-2                                 8930       7675.45          8540   \n",
       "t-9                                 9835       7675.45         11881   \n",
       "t-8                              7675.45         11881         10650   \n",
       "t-6                                10650          4498       7675.45   \n",
       "t-15                             7675.45         12353         10520   \n",
       "t-21                                7368          6858          5890   \n",
       "t-13                               10520          9479         10529   \n",
       "t-20                                6858          5890          6913   \n",
       "MeanStoreSales                   7682.75       7682.75       7682.75   \n",
       "MeanRegionSales                  4698.57       4698.57       4698.57   \n",
       "D_Day_of_week                          0             1             2   \n",
       "_NumberOfSales                      8540         10364          4676   \n",
       "\n",
       "                                     652           653  \n",
       "StoreID                             1000          1000  \n",
       "Date                          2018-01-04    2018-01-05  \n",
       "IsHoliday                              1             0  \n",
       "IsOpen                                 0             1  \n",
       "HasPromotions                          0             0  \n",
       "StoreType                   Hyper Market  Hyper Market  \n",
       "AssortmentType                   General       General  \n",
       "NearestCompetitor                    326           326  \n",
       "Region                                 7             7  \n",
       "NumberOfCustomers                622.769           536  \n",
       "NumberOfSales                    7675.45          6267  \n",
       "Region_AreaKM2                      9643          9643  \n",
       "Region_GDP                         17130         17130  \n",
       "Region_PopulationK                  2770          2770  \n",
       "CloudCover                             7             6  \n",
       "Events                               Fog          Rain  \n",
       "Max_Dew_PointC                         2             3  \n",
       "Max_Humidity                         100           100  \n",
       "Max_Sea_Level_PressurehPa           1038          1035  \n",
       "Max_TemperatureC                       3             4  \n",
       "Max_VisibilityKm                     673           674  \n",
       "Max_Wind_SpeedKm_h                    19            29  \n",
       "Mean_Dew_PointC                        2             1  \n",
       "Mean_Humidity                         96            91  \n",
       "Mean_Sea_Level_PressurehPa          1036          1030  \n",
       "Mean_TemperatureC                      2             2  \n",
       "Mean_VisibilityKm                    673           674  \n",
       "Mean_Wind_SpeedKm_h                    6            21  \n",
       "Min_Dew_PointC                         1            -1  \n",
       "Min_Humidity                          87            84  \n",
       "Min_Sea_Level_PressurehPa           1033          1028  \n",
       "Min_TemperatureC                       1             0  \n",
       "Min_VisibilitykM                     673           674  \n",
       "Precipitationmm                        0           6.1  \n",
       "WindDirDegrees                       205           206  \n",
       "D_Day                                  4             5  \n",
       "D_DayOfYear                            4             5  \n",
       "D_Month                                1             1  \n",
       "D_Year                              2018          2018  \n",
       "t-14                               10529          9050  \n",
       "t-7                              7675.45       7675.45  \n",
       "t-1                                 4676       7675.45  \n",
       "t-2                                10364          4676  \n",
       "t-9                                10650          4498  \n",
       "t-8                                 4498       7675.45  \n",
       "t-6                              7675.45          8930  \n",
       "t-15                                9479         10529  \n",
       "t-21                                6913          7282  \n",
       "t-13                                9050          9835  \n",
       "t-20                                7282          9179  \n",
       "MeanStoreSales                   7682.75       7682.75  \n",
       "MeanRegionSales                  4698.57       4698.57  \n",
       "D_Day_of_week                          3             4  \n",
       "_NumberOfSales                   7675.45          6267  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import modify_dataset as md\n",
    "train = md.get_fake_train(df)\n",
    "df_test = md.get_fake_test(df)\n",
    "\n",
    "df_test['_NumberOfSales'] = df_test['NumberOfSales']\n",
    "#df_test.drop(['NumberOfSales'], axis=1, inplace=True)\n",
    "\n",
    "df_test.head().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     43816\n",
       "unique      749\n",
       "top        1748\n",
       "freq         59\n",
       "Name: StoreID, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_cat = df_test['StoreID']\n",
    "dataset_cat.astype('object').describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>t-14</th>\n",
       "      <th>t-7</th>\n",
       "      <th>t-1</th>\n",
       "      <th>t-2</th>\n",
       "      <th>t-9</th>\n",
       "      <th>t-8</th>\n",
       "      <th>t-6</th>\n",
       "      <th>t-15</th>\n",
       "      <th>t-21</th>\n",
       "      <th>t-13</th>\n",
       "      <th>t-20</th>\n",
       "      <th>MeanStoreSales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8804.000000</td>\n",
       "      <td>6014.000000</td>\n",
       "      <td>8155.0</td>\n",
       "      <td>9183.0</td>\n",
       "      <td>6348.0</td>\n",
       "      <td>4738.0</td>\n",
       "      <td>6087.000000</td>\n",
       "      <td>9019.0</td>\n",
       "      <td>5676.000000</td>\n",
       "      <td>7823.000000</td>\n",
       "      <td>8111.000000</td>\n",
       "      <td>7682.751625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7823.000000</td>\n",
       "      <td>6087.000000</td>\n",
       "      <td>7109.0</td>\n",
       "      <td>8155.0</td>\n",
       "      <td>4738.0</td>\n",
       "      <td>6014.0</td>\n",
       "      <td>6962.000000</td>\n",
       "      <td>8804.0</td>\n",
       "      <td>8111.000000</td>\n",
       "      <td>7989.000000</td>\n",
       "      <td>8300.000000</td>\n",
       "      <td>7682.751625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7989.000000</td>\n",
       "      <td>6962.000000</td>\n",
       "      <td>8041.0</td>\n",
       "      <td>7109.0</td>\n",
       "      <td>6014.0</td>\n",
       "      <td>6087.0</td>\n",
       "      <td>6618.000000</td>\n",
       "      <td>7823.0</td>\n",
       "      <td>8300.000000</td>\n",
       "      <td>5895.000000</td>\n",
       "      <td>7154.000000</td>\n",
       "      <td>7682.751625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5895.000000</td>\n",
       "      <td>6618.000000</td>\n",
       "      <td>7890.0</td>\n",
       "      <td>8041.0</td>\n",
       "      <td>6087.0</td>\n",
       "      <td>6962.0</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7989.0</td>\n",
       "      <td>7154.000000</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7682.751625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>6005.0</td>\n",
       "      <td>7890.0</td>\n",
       "      <td>6962.0</td>\n",
       "      <td>6618.0</td>\n",
       "      <td>9183.000000</td>\n",
       "      <td>5895.0</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>6348.000000</td>\n",
       "      <td>10110.000000</td>\n",
       "      <td>7682.751625</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          t-14          t-7     t-1     t-2     t-9     t-8          t-6  \\\n",
       "0  8804.000000  6014.000000  8155.0  9183.0  6348.0  4738.0  6087.000000   \n",
       "1  7823.000000  6087.000000  7109.0  8155.0  4738.0  6014.0  6962.000000   \n",
       "2  7989.000000  6962.000000  8041.0  7109.0  6014.0  6087.0  6618.000000   \n",
       "3  5895.000000  6618.000000  7890.0  8041.0  6087.0  6962.0  7675.446488   \n",
       "4  7675.446488  7675.446488  6005.0  7890.0  6962.0  6618.0  9183.000000   \n",
       "\n",
       "     t-15         t-21         t-13          t-20  MeanStoreSales  \n",
       "0  9019.0  5676.000000  7823.000000   8111.000000     7682.751625  \n",
       "1  8804.0  8111.000000  7989.000000   8300.000000     7682.751625  \n",
       "2  7823.0  8300.000000  5895.000000   7154.000000     7682.751625  \n",
       "3  7989.0  7154.000000  7675.446488   7675.446488     7682.751625  \n",
       "4  5895.0  7675.446488  6348.000000  10110.000000     7682.751625  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#features = ['Max_Dew_PointC','Max_Humidity','Max_Sea_Level_PressurehPa','Max_TemperatureC',\n",
    "#           'Max_Wind_SpeedKm_h','Mean_Dew_PointC','Mean_Humidity','Mean_Sea_Level_PressurehPa',\n",
    "#            'Mean_TemperatureC','Mean_Wind_SpeedKm_h','Min_Dew_PointC','Min_Humidity',\n",
    "#           'Min_Sea_Level_PressurehPa','Min_TemperatureC','Precipitationmm','Mean_VisibilityKm']\n",
    "# Separating out the features\n",
    "\n",
    "features = ['t-14','t-7','t-1','t-2','t-9','t-8','t-6','t-15','t-21','t-13','t-20','MeanStoreSales']\n",
    "\n",
    "x = train[features]\n",
    "# Separating out the target\n",
    "y = train['NumberOfSales']\n",
    "x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StoreID</th>\n",
       "      <th>Date</th>\n",
       "      <th>NumberOfSales</th>\n",
       "      <th>_NumberOfSales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>649</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>8540.000000</td>\n",
       "      <td>8540.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>10364.000000</td>\n",
       "      <td>10364.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>4676.000000</td>\n",
       "      <td>4676.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7675.446488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-05</td>\n",
       "      <td>6267.000000</td>\n",
       "      <td>6267.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-06</td>\n",
       "      <td>5953.000000</td>\n",
       "      <td>5953.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>655</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-07</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7675.446488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>656</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-08</td>\n",
       "      <td>9075.000000</td>\n",
       "      <td>9075.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-09</td>\n",
       "      <td>7856.000000</td>\n",
       "      <td>7856.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>658</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-10</td>\n",
       "      <td>7423.000000</td>\n",
       "      <td>7423.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-11</td>\n",
       "      <td>6283.000000</td>\n",
       "      <td>6283.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-12</td>\n",
       "      <td>7213.000000</td>\n",
       "      <td>7213.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-13</td>\n",
       "      <td>5738.000000</td>\n",
       "      <td>5738.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>662</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-14</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7675.446488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-15</td>\n",
       "      <td>8902.000000</td>\n",
       "      <td>8902.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     StoreID        Date  NumberOfSales  _NumberOfSales\n",
       "649     1000  2018-01-01    8540.000000     8540.000000\n",
       "650     1000  2018-01-02   10364.000000    10364.000000\n",
       "651     1000  2018-01-03    4676.000000     4676.000000\n",
       "652     1000  2018-01-04    7675.446488     7675.446488\n",
       "653     1000  2018-01-05    6267.000000     6267.000000\n",
       "654     1000  2018-01-06    5953.000000     5953.000000\n",
       "655     1000  2018-01-07    7675.446488     7675.446488\n",
       "656     1000  2018-01-08    9075.000000     9075.000000\n",
       "657     1000  2018-01-09    7856.000000     7856.000000\n",
       "658     1000  2018-01-10    7423.000000     7423.000000\n",
       "659     1000  2018-01-11    6283.000000     6283.000000\n",
       "660     1000  2018-01-12    7213.000000     7213.000000\n",
       "661     1000  2018-01-13    5738.000000     5738.000000\n",
       "662     1000  2018-01-14    7675.446488     7675.446488\n",
       "663     1000  2018-01-15    8902.000000     8902.000000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test[['StoreID', 'Date', 'NumberOfSales', '_NumberOfSales']].head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "days = test['Date'].loc[:'Date'].values\n",
    "days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.46990029,  0.10011236,  0.40130522,  0.1202943 , -0.0995176 ,\n",
       "       -0.11358087, -0.04022331, -0.2442403 ,  0.11728318,  0.05857282,\n",
       "       -0.05393229,  0.2933977 ])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn.linear_model\n",
    "model = sklearn.linear_model.LinearRegression(fit_intercept=True, normalize=True, copy_X=False, n_jobs=-1).fit(x,y)\n",
    "model.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "row = test.loc[(test['Date'].values == '2018-01-01')]\n",
    "pred = model.predict(row[features])\n",
    "pred = pd.DataFrame(pred,columns=['NumberOfSales'],index=row['StoreID'].values)\n",
    "pred['Date']=row['Date'].values\n",
    "pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "df_dbd = pd.DataFrame(columns=['NumberOfSales','Date'])\n",
    "df_dbd = pd.concat((df_dbd,pred))\n",
    "df_dbd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['_NumberOfSales'] = model.predict(df_test[features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StoreID</th>\n",
       "      <th>Date</th>\n",
       "      <th>NumberOfSales</th>\n",
       "      <th>_NumberOfSales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>649</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>8540.000000</td>\n",
       "      <td>10326.681023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>10364.000000</td>\n",
       "      <td>8311.817340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>4676.000000</td>\n",
       "      <td>7976.716576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7813.966539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-05</td>\n",
       "      <td>6267.000000</td>\n",
       "      <td>7569.811723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>654</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-06</td>\n",
       "      <td>5953.000000</td>\n",
       "      <td>8132.446127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>655</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-07</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>6145.358358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>656</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-08</td>\n",
       "      <td>9075.000000</td>\n",
       "      <td>9908.634903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>657</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-09</td>\n",
       "      <td>7856.000000</td>\n",
       "      <td>8990.742676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>658</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-10</td>\n",
       "      <td>7423.000000</td>\n",
       "      <td>5103.837870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>659</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-11</td>\n",
       "      <td>6283.000000</td>\n",
       "      <td>8803.458426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-12</td>\n",
       "      <td>7213.000000</td>\n",
       "      <td>7472.510231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-13</td>\n",
       "      <td>5738.000000</td>\n",
       "      <td>8193.914934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>662</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-14</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>6680.453860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-15</td>\n",
       "      <td>8902.000000</td>\n",
       "      <td>8684.418173</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     StoreID        Date  NumberOfSales  _NumberOfSales\n",
       "649     1000  2018-01-01    8540.000000    10326.681023\n",
       "650     1000  2018-01-02   10364.000000     8311.817340\n",
       "651     1000  2018-01-03    4676.000000     7976.716576\n",
       "652     1000  2018-01-04    7675.446488     7813.966539\n",
       "653     1000  2018-01-05    6267.000000     7569.811723\n",
       "654     1000  2018-01-06    5953.000000     8132.446127\n",
       "655     1000  2018-01-07    7675.446488     6145.358358\n",
       "656     1000  2018-01-08    9075.000000     9908.634903\n",
       "657     1000  2018-01-09    7856.000000     8990.742676\n",
       "658     1000  2018-01-10    7423.000000     5103.837870\n",
       "659     1000  2018-01-11    6283.000000     8803.458426\n",
       "660     1000  2018-01-12    7213.000000     7472.510231\n",
       "661     1000  2018-01-13    5738.000000     8193.914934\n",
       "662     1000  2018-01-14    7675.446488     6680.453860\n",
       "663     1000  2018-01-15    8902.000000     8684.418173"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test[['StoreID', 'Date', 'NumberOfSales', '_NumberOfSales']].head(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of regions identified: 11\n",
      "BIP total error: 0.047672183320945985\n"
     ]
    }
   ],
   "source": [
    "from BIP import get_BIP_error\n",
    "\n",
    "error = get_BIP_error(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(43816, 54)\n",
      "RSS: 42634632288.560196\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StoreID</th>\n",
       "      <th>Date</th>\n",
       "      <th>IsHoliday</th>\n",
       "      <th>IsOpen</th>\n",
       "      <th>HasPromotions</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>AssortmentType</th>\n",
       "      <th>NearestCompetitor</th>\n",
       "      <th>Region</th>\n",
       "      <th>NumberOfCustomers</th>\n",
       "      <th>...</th>\n",
       "      <th>t-6</th>\n",
       "      <th>t-15</th>\n",
       "      <th>t-21</th>\n",
       "      <th>t-13</th>\n",
       "      <th>t-20</th>\n",
       "      <th>MeanStoreSales</th>\n",
       "      <th>MeanRegionSales</th>\n",
       "      <th>D_Day_of_week</th>\n",
       "      <th>_NumberOfSales</th>\n",
       "      <th>sqrd_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>649</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>642.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10650.000000</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7368.0</td>\n",
       "      <td>10520.0</td>\n",
       "      <td>6858.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>0</td>\n",
       "      <td>10326.681023</td>\n",
       "      <td>3.192229e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>746.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4498.000000</td>\n",
       "      <td>12353.000000</td>\n",
       "      <td>6858.0</td>\n",
       "      <td>9479.0</td>\n",
       "      <td>5890.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>1</td>\n",
       "      <td>8311.817340</td>\n",
       "      <td>4.211454e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>333.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>10520.000000</td>\n",
       "      <td>5890.0</td>\n",
       "      <td>10529.0</td>\n",
       "      <td>6913.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>2</td>\n",
       "      <td>7976.716576</td>\n",
       "      <td>1.089473e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>622.769231</td>\n",
       "      <td>...</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>9479.000000</td>\n",
       "      <td>6913.0</td>\n",
       "      <td>9050.0</td>\n",
       "      <td>7282.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>3</td>\n",
       "      <td>7813.966539</td>\n",
       "      <td>1.918780e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>536.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>8930.000000</td>\n",
       "      <td>10529.000000</td>\n",
       "      <td>7282.0</td>\n",
       "      <td>9835.0</td>\n",
       "      <td>9179.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>4</td>\n",
       "      <td>7569.811723</td>\n",
       "      <td>1.697318e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     StoreID        Date  IsHoliday  IsOpen  HasPromotions     StoreType  \\\n",
       "649     1000  2018-01-01          0       1              0  Hyper Market   \n",
       "650     1000  2018-01-02          0       1              0  Hyper Market   \n",
       "651     1000  2018-01-03          0       1              0  Hyper Market   \n",
       "652     1000  2018-01-04          1       0              0  Hyper Market   \n",
       "653     1000  2018-01-05          0       1              0  Hyper Market   \n",
       "\n",
       "    AssortmentType  NearestCompetitor  Region  NumberOfCustomers  \\\n",
       "649        General                326       7         642.000000   \n",
       "650        General                326       7         746.000000   \n",
       "651        General                326       7         333.000000   \n",
       "652        General                326       7         622.769231   \n",
       "653        General                326       7         536.000000   \n",
       "\n",
       "         ...                t-6          t-15    t-21     t-13    t-20  \\\n",
       "649      ...       10650.000000   7675.446488  7368.0  10520.0  6858.0   \n",
       "650      ...        4498.000000  12353.000000  6858.0   9479.0  5890.0   \n",
       "651      ...        7675.446488  10520.000000  5890.0  10529.0  6913.0   \n",
       "652      ...        7675.446488   9479.000000  6913.0   9050.0  7282.0   \n",
       "653      ...        8930.000000  10529.000000  7282.0   9835.0  9179.0   \n",
       "\n",
       "    MeanStoreSales  MeanRegionSales  D_Day_of_week  _NumberOfSales  \\\n",
       "649    7682.751625      4698.572717              0    10326.681023   \n",
       "650    7682.751625      4698.572717              1     8311.817340   \n",
       "651    7682.751625      4698.572717              2     7976.716576   \n",
       "652    7682.751625      4698.572717              3     7813.966539   \n",
       "653    7682.751625      4698.572717              4     7569.811723   \n",
       "\n",
       "        sqrd_diff  \n",
       "649  3.192229e+06  \n",
       "650  4.211454e+06  \n",
       "651  1.089473e+07  \n",
       "652  1.918780e+04  \n",
       "653  1.697318e+06  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculationg R2\n",
    "\n",
    "# calculate RSS\n",
    "df_RSS = df_test\n",
    "print(df_RSS.shape)\n",
    "\n",
    "df_RSS['sqrd_diff'] = df_RSS.apply(lambda x: (x['NumberOfSales'] - x['_NumberOfSales'])**2, axis=1)\n",
    "\n",
    "RSS = df_RSS['sqrd_diff'].sum()\n",
    "print('RSS:', RSS)\n",
    "df_RSS.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NS_mean 4773.635883889883\n",
      "TSS: 163118336287.97296\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>StoreID</th>\n",
       "      <th>Date</th>\n",
       "      <th>IsHoliday</th>\n",
       "      <th>IsOpen</th>\n",
       "      <th>HasPromotions</th>\n",
       "      <th>StoreType</th>\n",
       "      <th>AssortmentType</th>\n",
       "      <th>NearestCompetitor</th>\n",
       "      <th>Region</th>\n",
       "      <th>NumberOfCustomers</th>\n",
       "      <th>...</th>\n",
       "      <th>t-6</th>\n",
       "      <th>t-15</th>\n",
       "      <th>t-21</th>\n",
       "      <th>t-13</th>\n",
       "      <th>t-20</th>\n",
       "      <th>MeanStoreSales</th>\n",
       "      <th>MeanRegionSales</th>\n",
       "      <th>D_Day_of_week</th>\n",
       "      <th>_NumberOfSales</th>\n",
       "      <th>sqrd_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>649</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>642.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>10650.000000</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>7368.0</td>\n",
       "      <td>10520.0</td>\n",
       "      <td>6858.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>0</td>\n",
       "      <td>10326.681023</td>\n",
       "      <td>3.083631e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>746.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4498.000000</td>\n",
       "      <td>12353.000000</td>\n",
       "      <td>6858.0</td>\n",
       "      <td>9479.0</td>\n",
       "      <td>5890.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>1</td>\n",
       "      <td>8311.817340</td>\n",
       "      <td>1.251873e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>333.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>10520.000000</td>\n",
       "      <td>5890.0</td>\n",
       "      <td>10529.0</td>\n",
       "      <td>6913.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>2</td>\n",
       "      <td>7976.716576</td>\n",
       "      <td>1.025973e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>622.769231</td>\n",
       "      <td>...</td>\n",
       "      <td>7675.446488</td>\n",
       "      <td>9479.000000</td>\n",
       "      <td>6913.0</td>\n",
       "      <td>9050.0</td>\n",
       "      <td>7282.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>3</td>\n",
       "      <td>7813.966539</td>\n",
       "      <td>9.243610e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>1000</td>\n",
       "      <td>2018-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hyper Market</td>\n",
       "      <td>General</td>\n",
       "      <td>326</td>\n",
       "      <td>7</td>\n",
       "      <td>536.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>8930.000000</td>\n",
       "      <td>10529.000000</td>\n",
       "      <td>7282.0</td>\n",
       "      <td>9835.0</td>\n",
       "      <td>9179.0</td>\n",
       "      <td>7682.751625</td>\n",
       "      <td>4698.572717</td>\n",
       "      <td>4</td>\n",
       "      <td>7569.811723</td>\n",
       "      <td>7.818599e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     StoreID        Date  IsHoliday  IsOpen  HasPromotions     StoreType  \\\n",
       "649     1000  2018-01-01          0       1              0  Hyper Market   \n",
       "650     1000  2018-01-02          0       1              0  Hyper Market   \n",
       "651     1000  2018-01-03          0       1              0  Hyper Market   \n",
       "652     1000  2018-01-04          1       0              0  Hyper Market   \n",
       "653     1000  2018-01-05          0       1              0  Hyper Market   \n",
       "\n",
       "    AssortmentType  NearestCompetitor  Region  NumberOfCustomers  \\\n",
       "649        General                326       7         642.000000   \n",
       "650        General                326       7         746.000000   \n",
       "651        General                326       7         333.000000   \n",
       "652        General                326       7         622.769231   \n",
       "653        General                326       7         536.000000   \n",
       "\n",
       "         ...                t-6          t-15    t-21     t-13    t-20  \\\n",
       "649      ...       10650.000000   7675.446488  7368.0  10520.0  6858.0   \n",
       "650      ...        4498.000000  12353.000000  6858.0   9479.0  5890.0   \n",
       "651      ...        7675.446488  10520.000000  5890.0  10529.0  6913.0   \n",
       "652      ...        7675.446488   9479.000000  6913.0   9050.0  7282.0   \n",
       "653      ...        8930.000000  10529.000000  7282.0   9835.0  9179.0   \n",
       "\n",
       "    MeanStoreSales  MeanRegionSales  D_Day_of_week  _NumberOfSales  \\\n",
       "649    7682.751625      4698.572717              0    10326.681023   \n",
       "650    7682.751625      4698.572717              1     8311.817340   \n",
       "651    7682.751625      4698.572717              2     7976.716576   \n",
       "652    7682.751625      4698.572717              3     7813.966539   \n",
       "653    7682.751625      4698.572717              4     7569.811723   \n",
       "\n",
       "        sqrd_diff  \n",
       "649  3.083631e+07  \n",
       "650  1.251873e+07  \n",
       "651  1.025973e+07  \n",
       "652  9.243610e+06  \n",
       "653  7.818599e+06  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate TSS\n",
    "\n",
    "df_TSS = df_test\n",
    "NS_mean = df_TSS['NumberOfSales'].mean()\n",
    "print('NS_mean', NS_mean)\n",
    "\n",
    "df_TSS['sqrd_diff'] = df_TSS.apply(lambda x:  (x['_NumberOfSales'] - NS_mean)**2, axis=1)\n",
    "TSS = df_TSS['sqrd_diff'].sum()\n",
    "print('TSS:', TSS)\n",
    "df_TSS.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RSS: 42634632288.560196    TSS: 163118336287.97296\n",
      "R_2: 0.7386275923431931\n"
     ]
    }
   ],
   "source": [
    "# calculate R2\n",
    "print(\"RSS:\", RSS, \"   TSS:\", TSS)\n",
    "\n",
    "R2 = 1 - (RSS / TSS)\n",
    "print('R_2:', R2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}