DYD_MS-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/MS-day/2017-jan-day-MS.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4fe01e07e4cc830b6998642dd0327a85d13b615a</td>\n",
       "      <td>2017-01-28 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>704</td>\n",
       "      <td>704</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4fe01e07e4cc830b6998642dd0327a85d13b615a</td>\n",
       "      <td>2017-01-08 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>738</td>\n",
       "      <td>738</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6a4e92ba22307727b97e8d8deef28f5373f93ade</td>\n",
       "      <td>2017-01-29 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>674</td>\n",
       "      <td>737</td>\n",
       "      <td>686</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e46f291ea1b93a73e8c0dbfd7a0ac8c1fd1f0ad7</td>\n",
       "      <td>2017-01-15 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7e94fec7feacae0a160256474e0b2b5b4fc73598</td>\n",
       "      <td>2017-01-13 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>754</td>\n",
       "      <td>704</td>\n",
       "      <td>MS</td>\n",
       "      <td>Olive Branch</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63484</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-01-29 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63485</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-01-06 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63486</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-01-24 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63487</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-01-10 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63488</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-01-06 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60650 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      4fe01e07e4cc830b6998642dd0327a85d13b615a  2017-01-28 14:10:00 UTC   \n",
       "1      4fe01e07e4cc830b6998642dd0327a85d13b615a  2017-01-08 19:50:00 UTC   \n",
       "2      6a4e92ba22307727b97e8d8deef28f5373f93ade  2017-01-29 14:55:00 UTC   \n",
       "3      e46f291ea1b93a73e8c0dbfd7a0ac8c1fd1f0ad7  2017-01-15 18:35:00 UTC   \n",
       "4      7e94fec7feacae0a160256474e0b2b5b4fc73598  2017-01-13 16:50:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "63484  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-01-29 12:50:00 UTC   \n",
       "63485  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-01-06 15:35:00 UTC   \n",
       "63486  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-01-24 17:45:00 UTC   \n",
       "63487  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-01-10 12:50:00 UTC   \n",
       "63488  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-01-06 13:30:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               707                      704   \n",
       "1         heat          hold               734                      738   \n",
       "2         auto          auto               674                      737   \n",
       "3         auto          auto               686                      685   \n",
       "4         auto          auto               703                      754   \n",
       "...        ...           ...               ...                      ...   \n",
       "63484     heat          auto               761                      760   \n",
       "63485     heat          auto               761                      760   \n",
       "63486     heat          auto               760                      760   \n",
       "63487     heat          auto               759                      760   \n",
       "63488     heat          auto               756                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                          704            MS       Jackson   \n",
       "1                          738            MS       Jackson   \n",
       "2                          686            MS        Oxford   \n",
       "3                          635            MS       Brandon   \n",
       "4                          704            MS  Olive Branch   \n",
       "...                        ...           ...           ...   \n",
       "63484                      760            MS    Mooreville   \n",
       "63485                      760            MS    Mooreville   \n",
       "63486                      760            MS    Mooreville   \n",
       "63487                      760            MS    Mooreville   \n",
       "63488                      760            MS    Mooreville   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       20             False         False            False   \n",
       "1                       20             False         False            False   \n",
       "2                       35              True         False             True   \n",
       "3                       25             False         False            False   \n",
       "4                       15             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "63484                   10             False         False            False   \n",
       "63485                   10             False         False            False   \n",
       "63486                   10             False         False            False   \n",
       "63487                   10             False         False            False   \n",
       "63488                   10             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                      Electric  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "63484                       Gas  \n",
       "63485                       Gas  \n",
       "63486                       Gas  \n",
       "63487                       Gas  \n",
       "63488                       Gas  \n",
       "\n",
       "[60650 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">01194744830da7bbc9603d51735c58faef66f72e</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">cool</th>\n",
       "      <th>auto</th>\n",
       "      <th>Ocean Springs</th>\n",
       "      <td>704.883117</td>\n",
       "      <td>737.571429</td>\n",
       "      <td>683.116883</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Ocean Springs</th>\n",
       "      <td>721.615385</td>\n",
       "      <td>732.977667</td>\n",
       "      <td>732.977667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Ocean Springs</th>\n",
       "      <td>692.216667</td>\n",
       "      <td>708.183333</td>\n",
       "      <td>705.283333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Ocean Springs</th>\n",
       "      <td>714.270330</td>\n",
       "      <td>691.597802</td>\n",
       "      <td>691.597802</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>024d1624a424d57f4cdc19a5fa9aa6d90739ef33</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Horn Lake</th>\n",
       "      <td>712.636364</td>\n",
       "      <td>755.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fe9bc276b524605ee152e4051285e4e080e77a3f</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Tupelo</th>\n",
       "      <td>700.236842</td>\n",
       "      <td>714.005263</td>\n",
       "      <td>702.510526</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Tupelo</th>\n",
       "      <td>693.190871</td>\n",
       "      <td>688.796680</td>\n",
       "      <td>688.796680</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">ffe938566049e7ad2eb8c1726ec2215bfda979d7</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">2017</th>\n",
       "      <th>cool</th>\n",
       "      <th>auto</th>\n",
       "      <th>Mooreville</th>\n",
       "      <td>755.084906</td>\n",
       "      <td>744.207547</td>\n",
       "      <td>747.547170</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Mooreville</th>\n",
       "      <td>756.862367</td>\n",
       "      <td>748.020823</td>\n",
       "      <td>748.158964</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Mooreville</th>\n",
       "      <td>756.477612</td>\n",
       "      <td>748.369403</td>\n",
       "      <td>748.559701</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                          Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                  \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs            704.883117   \n",
       "                                                             hold          Ocean Springs            721.615385   \n",
       "                                                    heat     auto          Ocean Springs            692.216667   \n",
       "                                                             hold          Ocean Springs            714.270330   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                712.636364   \n",
       "...                                                                                                        ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                   700.236842   \n",
       "                                                             hold          Tupelo                   693.190871   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville               755.084906   \n",
       "                                                    heat     auto          Mooreville               756.862367   \n",
       "                                                             hold          Mooreville               756.477612   \n",
       "\n",
       "                                                                                          TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs                   737.571429   \n",
       "                                                             hold          Ocean Springs                   732.977667   \n",
       "                                                    heat     auto          Ocean Springs                   708.183333   \n",
       "                                                             hold          Ocean Springs                   691.597802   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                       755.000000   \n",
       "...                                                                                                               ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                          714.005263   \n",
       "                                                             hold          Tupelo                          688.796680   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville                      744.207547   \n",
       "                                                    heat     auto          Mooreville                      748.020823   \n",
       "                                                             hold          Mooreville                      748.369403   \n",
       "\n",
       "                                                                                          TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs                   683.116883   \n",
       "                                                             hold          Ocean Springs                   732.977667   \n",
       "                                                    heat     auto          Ocean Springs                   705.283333   \n",
       "                                                             hold          Ocean Springs                   691.597802   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                       695.000000   \n",
       "...                                                                                                               ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                          702.510526   \n",
       "                                                             hold          Tupelo                          688.796680   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville                      747.547170   \n",
       "                                                    heat     auto          Mooreville                      748.158964   \n",
       "                                                             hold          Mooreville                      748.559701   \n",
       "\n",
       "                                                                                          Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs                  0.0   \n",
       "                                                             hold          Ocean Springs                  0.0   \n",
       "                                                    heat     auto          Ocean Springs                  0.0   \n",
       "                                                             hold          Ocean Springs                  0.0   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                     26.0   \n",
       "...                                                                                                       ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                        75.0   \n",
       "                                                             hold          Tupelo                        75.0   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville                    10.0   \n",
       "                                                    heat     auto          Mooreville                    10.0   \n",
       "                                                             hold          Mooreville                    10.0   \n",
       "\n",
       "                                                                                          allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                              \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs             False   \n",
       "                                                             hold          Ocean Springs             False   \n",
       "                                                    heat     auto          Ocean Springs             False   \n",
       "                                                             hold          Ocean Springs             False   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                  True   \n",
       "...                                                                                                    ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                    False   \n",
       "                                                             hold          Tupelo                    False   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville                False   \n",
       "                                                    heat     auto          Mooreville                False   \n",
       "                                                             hold          Mooreville                False   \n",
       "\n",
       "                                                                                          Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs         False   \n",
       "                                                             hold          Ocean Springs         False   \n",
       "                                                    heat     auto          Ocean Springs         False   \n",
       "                                                             hold          Ocean Springs         False   \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake             False   \n",
       "...                                                                                                ...   \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                False   \n",
       "                                                             hold          Tupelo                False   \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville            False   \n",
       "                                                    heat     auto          Mooreville            False   \n",
       "                                                             hold          Mooreville            False   \n",
       "\n",
       "                                                                                          Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "01194744830da7bbc9603d51735c58faef66f72e Jan   2017 cool     auto          Ocean Springs             True  \n",
       "                                                             hold          Ocean Springs             True  \n",
       "                                                    heat     auto          Ocean Springs             True  \n",
       "                                                             hold          Ocean Springs             True  \n",
       "024d1624a424d57f4cdc19a5fa9aa6d90739ef33 Jan   2017 auto     hold          Horn Lake                 True  \n",
       "...                                                                                                   ...  \n",
       "fe9bc276b524605ee152e4051285e4e080e77a3f Jan   2017 heat     auto          Tupelo                   False  \n",
       "                                                             hold          Tupelo                   False  \n",
       "ffe938566049e7ad2eb8c1726ec2215bfda979d7 Jan   2017 cool     auto          Mooreville               False  \n",
       "                                                    heat     auto          Mooreville               False  \n",
       "                                                             hold          Mooreville               False  \n",
       "\n",
       "[191 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/MS/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/MS-day/2018-jan-day-MS.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273ce90afa5e8188e6ff7544f58ac1484611ee8f</td>\n",
       "      <td>2018-01-12 10:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>83a4d060a27b70bfeb5a64d07cdc779aa2201fca</td>\n",
       "      <td>2018-01-10 15:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>273ce90afa5e8188e6ff7544f58ac1484611ee8f</td>\n",
       "      <td>2018-01-22 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>684</td>\n",
       "      <td>684</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9b718599ed03efd98e2b15948c86641a7cd4b249</td>\n",
       "      <td>2018-01-07 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>83a4d060a27b70bfeb5a64d07cdc779aa2201fca</td>\n",
       "      <td>2018-01-10 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213849</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-01-20 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213850</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-01-29 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213851</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-01-12 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>773</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213852</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-01-12 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213853</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-01-07 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>205829 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       273ce90afa5e8188e6ff7544f58ac1484611ee8f  2018-01-12 10:05:00 UTC   \n",
       "1       83a4d060a27b70bfeb5a64d07cdc779aa2201fca  2018-01-10 15:20:00 UTC   \n",
       "3       273ce90afa5e8188e6ff7544f58ac1484611ee8f  2018-01-22 17:50:00 UTC   \n",
       "4       9b718599ed03efd98e2b15948c86641a7cd4b249  2018-01-07 16:30:00 UTC   \n",
       "5       83a4d060a27b70bfeb5a64d07cdc779aa2201fca  2018-01-10 17:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "213849  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-01-20 15:15:00 UTC   \n",
       "213850  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-01-29 19:30:00 UTC   \n",
       "213851  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-01-12 18:35:00 UTC   \n",
       "213852  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-01-12 18:15:00 UTC   \n",
       "213853  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-01-07 18:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               700                      699   \n",
       "1          heat          hold               711                      716   \n",
       "3          heat          hold               684                      684   \n",
       "4          heat          hold               717                      745   \n",
       "5          heat          hold               712                      716   \n",
       "...         ...           ...               ...                      ...   \n",
       "213849     heat          auto               766                      760   \n",
       "213850     heat          auto               759                      760   \n",
       "213851     heat          hold               773                      760   \n",
       "213852     heat          hold               778                      760   \n",
       "213853     heat          auto               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           699            MS     Madison   \n",
       "1                           716            MS     Brandon   \n",
       "3                           684            MS     Madison   \n",
       "4                           745            MS     Madison   \n",
       "5                           716            MS     Brandon   \n",
       "...                         ...           ...         ...   \n",
       "213849                      760            MS  Mooreville   \n",
       "213850                      760            MS  Mooreville   \n",
       "213851                      760            MS  Mooreville   \n",
       "213852                      760            MS  Mooreville   \n",
       "213853                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "213849                   10             False         False            False   \n",
       "213850                   10             False         False            False   \n",
       "213851                   10             False         False            False   \n",
       "213852                   10             False         False            False   \n",
       "213853                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "213849                       Gas  \n",
       "213850                       Gas  \n",
       "213851                       Gas  \n",
       "213852                       Gas  \n",
       "213853                       Gas  \n",
       "\n",
       "[205829 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/MS/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/MS-day/2019-jan-day-MS.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-01-13 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>674</td>\n",
       "      <td>624</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-01-12 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>617</td>\n",
       "      <td>668</td>\n",
       "      <td>618</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7a0b2f12434fc5c99328cd9d20d8d54b0cfe335a</td>\n",
       "      <td>2019-01-12 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>649</td>\n",
       "      <td>650</td>\n",
       "      <td>626</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ecru</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71fc691a191d2bbea3ea5b823d23477631bb8ea4</td>\n",
       "      <td>2019-01-20 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>676</td>\n",
       "      <td>643</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ecru</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-01-12 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>668</td>\n",
       "      <td>618</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300815</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-01-27 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300816</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-01-28 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300817</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-01-27 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300818</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-01-28 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300819</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-01-05 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>290072 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d23049697dc10f7bda2995dcd45a259452e331c5  2019-01-13 14:35:00 UTC   \n",
       "1       d23049697dc10f7bda2995dcd45a259452e331c5  2019-01-12 12:55:00 UTC   \n",
       "2       7a0b2f12434fc5c99328cd9d20d8d54b0cfe335a  2019-01-12 18:15:00 UTC   \n",
       "3       71fc691a191d2bbea3ea5b823d23477631bb8ea4  2019-01-20 14:35:00 UTC   \n",
       "4       d23049697dc10f7bda2995dcd45a259452e331c5  2019-01-12 14:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "300815  f918d405d012b701c0cb10dd807539f0a731ceea  2019-01-27 13:05:00 UTC   \n",
       "300816  f918d405d012b701c0cb10dd807539f0a731ceea  2019-01-28 16:10:00 UTC   \n",
       "300817  f918d405d012b701c0cb10dd807539f0a731ceea  2019-01-27 13:25:00 UTC   \n",
       "300818  f918d405d012b701c0cb10dd807539f0a731ceea  2019-01-28 12:40:00 UTC   \n",
       "300819  f918d405d012b701c0cb10dd807539f0a731ceea  2019-01-05 12:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               667                      674   \n",
       "1          auto          hold               617                      668   \n",
       "2          heat          hold               649                      650   \n",
       "3          heat          hold               658                      676   \n",
       "4          auto          hold               655                      668   \n",
       "...         ...           ...               ...                      ...   \n",
       "300815     heat          auto               743                      740   \n",
       "300816     heat          hold               738                      740   \n",
       "300817     heat          auto               736                      740   \n",
       "300818     heat          hold               736                      740   \n",
       "300819     heat          hold               731                      740   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           624            MS          Belden   \n",
       "1                           618            MS          Belden   \n",
       "2                           626            MS            Ecru   \n",
       "3                           643            MS            Ecru   \n",
       "4                           618            MS          Belden   \n",
       "...                         ...           ...             ...   \n",
       "300815                      740            MS  Pass Christian   \n",
       "300816                      740            MS  Pass Christian   \n",
       "300817                      740            MS  Pass Christian   \n",
       "300818                      740            MS  Pass Christian   \n",
       "300819                      740            MS  Pass Christian   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        30             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "300815                   10             False         False             True   \n",
       "300816                   10             False         False             True   \n",
       "300817                   10             False         False             True   \n",
       "300818                   10             False         False             True   \n",
       "300819                   10             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "300815                  Electric  \n",
       "300816                  Electric  \n",
       "300817                  Electric  \n",
       "300818                  Electric  \n",
       "300819                  Electric  \n",
       "\n",
       "[290072 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/MS/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/MS-day/2020-jan-day-MS.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>96161eebbbcaaadc0c47b24f4815c16e9f7518ce</td>\n",
       "      <td>2020-01-27 13:45:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>699</td>\n",
       "      <td>662</td>\n",
       "      <td>MS</td>\n",
       "      <td>Kiln</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ddc9cd51c44a01e89ea06266d7d641a4bd735de2</td>\n",
       "      <td>2020-01-05 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>648</td>\n",
       "      <td>744</td>\n",
       "      <td>677</td>\n",
       "      <td>MS</td>\n",
       "      <td>Booneville</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16daf0ca4bfcc11cc2e4721db2c14c38559c0a92</td>\n",
       "      <td>2020-01-06 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>MS</td>\n",
       "      <td>Amory</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2020-01-22 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2020-01-22 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317683</th>\n",
       "      <td>7cc2be730bbe938e66cd541effe371e022c894d3</td>\n",
       "      <td>2020-01-23 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317684</th>\n",
       "      <td>7cc2be730bbe938e66cd541effe371e022c894d3</td>\n",
       "      <td>2020-01-23 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317687</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-01-14 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317688</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-01-14 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317689</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-01-24 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>308068 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       96161eebbbcaaadc0c47b24f4815c16e9f7518ce  2020-01-27 13:45:00 UTC   \n",
       "3       ddc9cd51c44a01e89ea06266d7d641a4bd735de2  2020-01-05 19:45:00 UTC   \n",
       "4       16daf0ca4bfcc11cc2e4721db2c14c38559c0a92  2020-01-06 12:40:00 UTC   \n",
       "6       f918d405d012b701c0cb10dd807539f0a731ceea  2020-01-22 18:05:00 UTC   \n",
       "7       f918d405d012b701c0cb10dd807539f0a731ceea  2020-01-22 13:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "317683  7cc2be730bbe938e66cd541effe371e022c894d3  2020-01-23 16:35:00 UTC   \n",
       "317684  7cc2be730bbe938e66cd541effe371e022c894d3  2020-01-23 18:25:00 UTC   \n",
       "317687  748c069371486cb31498d4fa5bec2a5949d72e83  2020-01-14 12:55:00 UTC   \n",
       "317688  748c069371486cb31498d4fa5bec2a5949d72e83  2020-01-14 12:35:00 UTC   \n",
       "317689  748c069371486cb31498d4fa5bec2a5949d72e83  2020-01-24 12:55:00 UTC   \n",
       "\n",
       "           HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0       auxHeatOnly          auto               677                      699   \n",
       "3              auto          auto               648                      744   \n",
       "4              heat          hold               688                      687   \n",
       "6              heat          auto               715                      743   \n",
       "7              heat          auto               712                      743   \n",
       "...             ...           ...               ...                      ...   \n",
       "317683         cool          hold               735                      740   \n",
       "317684         cool          hold               710                      740   \n",
       "317687         auto          hold               739                      810   \n",
       "317688         auto          hold               718                      810   \n",
       "317689         auto          hold               721                      810   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           662            MS            Kiln   \n",
       "3                           677            MS      Booneville   \n",
       "4                           687            MS           Amory   \n",
       "6                           717            MS  Pass Christian   \n",
       "7                           717            MS  Pass Christian   \n",
       "...                         ...           ...             ...   \n",
       "317683                      740            MS   Ocean Springs   \n",
       "317684                      740            MS   Ocean Springs   \n",
       "317687                      760            MS   Ocean Springs   \n",
       "317688                      760            MS   Ocean Springs   \n",
       "317689                      760            MS   Ocean Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        18              True         False             True   \n",
       "3                        60             False         False             True   \n",
       "4                        10              True         False             True   \n",
       "6                        10             False         False             True   \n",
       "7                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "317683                    0             False         False             True   \n",
       "317684                    0             False         False             True   \n",
       "317687                   19             False         False            False   \n",
       "317688                   19             False         False            False   \n",
       "317689                   19             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "6                       Electric  \n",
       "7                       Electric  \n",
       "...                          ...  \n",
       "317683                  Electric  \n",
       "317684                  Electric  \n",
       "317687                       Gas  \n",
       "317688                       Gas  \n",
       "317689                       Gas  \n",
       "\n",
       "[308068 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/MS/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/MS-day/2021-jan-day-MS.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cd632477aa8e4b117fe3fefc978149512e38a1be</td>\n",
       "      <td>2021-01-10 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>742</td>\n",
       "      <td>682</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4</td>\n",
       "      <td>2021-01-06 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>734</td>\n",
       "      <td>684</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e1d6f2c384d4e72dcc273a1d3d9625f298294878</td>\n",
       "      <td>2021-01-17 16:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>688</td>\n",
       "      <td>648</td>\n",
       "      <td>MS</td>\n",
       "      <td>Starkville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4</td>\n",
       "      <td>2021-01-07 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>794</td>\n",
       "      <td>744</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cd632477aa8e4b117fe3fefc978149512e38a1be</td>\n",
       "      <td>2021-01-24 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>742</td>\n",
       "      <td>682</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192780</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2021-01-11 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192781</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2021-01-11 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192782</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-01-10 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192783</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-01-10 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192784</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-01-10 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>783</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>188318 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cd632477aa8e4b117fe3fefc978149512e38a1be  2021-01-10 17:50:00 UTC   \n",
       "1       1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4  2021-01-06 15:55:00 UTC   \n",
       "2       e1d6f2c384d4e72dcc273a1d3d9625f298294878  2021-01-17 16:55:00 UTC   \n",
       "3       1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4  2021-01-07 17:40:00 UTC   \n",
       "5       cd632477aa8e4b117fe3fefc978149512e38a1be  2021-01-24 12:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "192780  dc1e40c795a411c773c78d408555dc894f11c103  2021-01-11 15:30:00 UTC   \n",
       "192781  dc1e40c795a411c773c78d408555dc894f11c103  2021-01-11 16:10:00 UTC   \n",
       "192782  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-01-10 18:25:00 UTC   \n",
       "192783  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-01-10 18:15:00 UTC   \n",
       "192784  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-01-10 18:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               683                      742   \n",
       "1          auto          hold               678                      734   \n",
       "2          heat          hold               678                      688   \n",
       "3          auto          hold               739                      794   \n",
       "5          auto          hold               684                      742   \n",
       "...         ...           ...               ...                      ...   \n",
       "192780     heat          hold               696                      760   \n",
       "192781     heat          hold               710                      760   \n",
       "192782     heat          hold               757                      760   \n",
       "192783     heat          hold               767                      760   \n",
       "192784     heat          hold               783                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           682            MS     Clinton   \n",
       "1                           684            MS      Tupelo   \n",
       "2                           648            MS  Starkville   \n",
       "3                           744            MS      Tupelo   \n",
       "5                           682            MS     Clinton   \n",
       "...                         ...           ...         ...   \n",
       "192780                      760            MS  New Albany   \n",
       "192781                      760            MS  New Albany   \n",
       "192782                      760            MS     Madison   \n",
       "192783                      760            MS     Madison   \n",
       "192784                      760            MS     Madison   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "192780                   10             False         False            False   \n",
       "192781                   10             False         False            False   \n",
       "192782                   10             False         False            False   \n",
       "192783                   10             False         False            False   \n",
       "192784                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "192780                       Gas  \n",
       "192781                       Gas  \n",
       "192782                       Gas  \n",
       "192783                       Gas  \n",
       "192784                       Gas  \n",
       "\n",
       "[188318 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/MS/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>704.883117</td>\n",
       "      <td>737.571429</td>\n",
       "      <td>683.116883</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>721.615385</td>\n",
       "      <td>732.977667</td>\n",
       "      <td>732.977667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>692.216667</td>\n",
       "      <td>708.183333</td>\n",
       "      <td>705.283333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>714.270330</td>\n",
       "      <td>691.597802</td>\n",
       "      <td>691.597802</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>024d1624a424d57f4cdc19a5fa9aa6d90739ef33</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>712.636364</td>\n",
       "      <td>755.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>676.365079</td>\n",
       "      <td>665.999142</td>\n",
       "      <td>665.999142</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>673.536585</td>\n",
       "      <td>687.585366</td>\n",
       "      <td>687.585366</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>303</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>714.558621</td>\n",
       "      <td>709.366897</td>\n",
       "      <td>708.497931</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>654.030303</td>\n",
       "      <td>651.151515</td>\n",
       "      <td>650.484848</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>738.126558</td>\n",
       "      <td>738.820410</td>\n",
       "      <td>738.820410</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1847 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   Jan  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   Jan  2017     cool   \n",
       "2    01194744830da7bbc9603d51735c58faef66f72e   Jan  2017     heat   \n",
       "3    01194744830da7bbc9603d51735c58faef66f72e   Jan  2017     heat   \n",
       "4    024d1624a424d57f4cdc19a5fa9aa6d90739ef33   Jan  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "301  fdfe0112380a8ee36c3dfe16899298a72c886d49   Jan  2021     heat   \n",
       "302  fe9bc276b524605ee152e4051285e4e080e77a3f   Jan  2021     cool   \n",
       "303  fe9bc276b524605ee152e4051285e4e080e77a3f   Jan  2021     heat   \n",
       "304  ff5551a641dc7446df1914428ded425dde839080   Jan  2021     heat   \n",
       "305  ffe938566049e7ad2eb8c1726ec2215bfda979d7   Jan  2021     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            704.883117   \n",
       "1            hold  Ocean Springs            721.615385   \n",
       "2            auto  Ocean Springs            692.216667   \n",
       "3            hold  Ocean Springs            714.270330   \n",
       "4            hold      Horn Lake            712.636364   \n",
       "..            ...            ...                   ...   \n",
       "301          hold        Jackson            676.365079   \n",
       "302          hold         Tupelo            673.536585   \n",
       "303          hold         Tupelo            714.558621   \n",
       "304          hold        Madison            654.030303   \n",
       "305          hold     Mooreville            738.126558   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     737.571429                   683.116883   \n",
       "1                     732.977667                   732.977667   \n",
       "2                     708.183333                   705.283333   \n",
       "3                     691.597802                   691.597802   \n",
       "4                     755.000000                   695.000000   \n",
       "..                           ...                          ...   \n",
       "301                   665.999142                   665.999142   \n",
       "302                   687.585366                   687.585366   \n",
       "303                   709.366897                   708.497931   \n",
       "304                   651.151515                   650.484848   \n",
       "305                   738.820410                   738.820410   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                    0.0             False         False             True  \n",
       "3                    0.0             False         False             True  \n",
       "4                   26.0              True         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "301                 30.0             False         False            False  \n",
       "302                 75.0             False         False            False  \n",
       "303                 75.0             False         False            False  \n",
       "304                 30.0             False         False            False  \n",
       "305                 10.0             False         False            False  \n",
       "\n",
       "[1847 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/jan/\" + file)\n",
    "    MS_jan = pd.concat([MS_jan, df])\n",
    "    \n",
    "MS_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_jan.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/MS-day/2017-feb-day-MS.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>83d9dc866298389693c96dd26cbe0bd6e3ad3725</td>\n",
       "      <td>2017-02-01 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>724</td>\n",
       "      <td>694</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c46244df34259c2508f601fd1ab7e084acce2f78</td>\n",
       "      <td>2017-02-10 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>708</td>\n",
       "      <td>708</td>\n",
       "      <td>MS</td>\n",
       "      <td>Hernando</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e2a5d9e8bff94f9fc2f5a8a8ed4dc57a80e0c7ef</td>\n",
       "      <td>2017-02-19 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>780</td>\n",
       "      <td>709</td>\n",
       "      <td>MS</td>\n",
       "      <td>Olive Branch</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8bb37505af1f12286cb960fed0eb3177c49e121b</td>\n",
       "      <td>2017-02-25 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>690</td>\n",
       "      <td>721</td>\n",
       "      <td>671</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0cc861d81926d70c036828385eb0e61b6e98dfaa</td>\n",
       "      <td>2017-02-09 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>834</td>\n",
       "      <td>762</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63035</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-02-04 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63036</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-02-09 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63037</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-02-03 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63038</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-02-09 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63039</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-02-04 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60662 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      83d9dc866298389693c96dd26cbe0bd6e3ad3725  2017-02-01 19:35:00 UTC   \n",
       "1      c46244df34259c2508f601fd1ab7e084acce2f78  2017-02-10 19:35:00 UTC   \n",
       "2      e2a5d9e8bff94f9fc2f5a8a8ed4dc57a80e0c7ef  2017-02-19 14:40:00 UTC   \n",
       "3      8bb37505af1f12286cb960fed0eb3177c49e121b  2017-02-25 16:45:00 UTC   \n",
       "5      0cc861d81926d70c036828385eb0e61b6e98dfaa  2017-02-09 12:50:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "63035  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-02-04 14:15:00 UTC   \n",
       "63036  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-02-09 13:20:00 UTC   \n",
       "63037  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-02-03 13:10:00 UTC   \n",
       "63038  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-02-09 16:20:00 UTC   \n",
       "63039  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-02-04 19:40:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          hold               729                      724   \n",
       "1         cool          hold               724                      708   \n",
       "2         auto          hold               734                      780   \n",
       "3         heat          auto               690                      721   \n",
       "5         auto          hold               746                      834   \n",
       "...        ...           ...               ...                      ...   \n",
       "63035     heat          auto               759                      760   \n",
       "63036     heat          auto               766                      760   \n",
       "63037     heat          auto               754                      760   \n",
       "63038     heat          auto               758                      760   \n",
       "63039     heat          auto               761                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                          694            MS       Brandon   \n",
       "1                          708            MS      Hernando   \n",
       "2                          709            MS  Olive Branch   \n",
       "3                          671            MS       Brandon   \n",
       "5                          762            MS       Madison   \n",
       "...                        ...           ...           ...   \n",
       "63035                      760            MS    Mooreville   \n",
       "63036                      760            MS    Mooreville   \n",
       "63037                      760            MS    Mooreville   \n",
       "63038                      760            MS    Mooreville   \n",
       "63039                      760            MS    Mooreville   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       15             False         False            False   \n",
       "1                        5              True         False            False   \n",
       "2                        0             False         False            False   \n",
       "3                       10             False         False            False   \n",
       "5                       10             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "63035                   10             False         False            False   \n",
       "63036                   10             False         False            False   \n",
       "63037                   10             False         False            False   \n",
       "63038                   10             False         False            False   \n",
       "63039                   10             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "5                           Gas  \n",
       "...                         ...  \n",
       "63035                       Gas  \n",
       "63036                       Gas  \n",
       "63037                       Gas  \n",
       "63038                       Gas  \n",
       "63039                       Gas  \n",
       "\n",
       "[60662 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/MS/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/MS-day/2018-feb-day-MS.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d74d5111738daa0acc3caa06e4e99ce488c26d14</td>\n",
       "      <td>2018-02-27 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>778</td>\n",
       "      <td>646</td>\n",
       "      <td>MS</td>\n",
       "      <td>Vancleave</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ad506c1a54f94de229685985f025564ac449d6a1</td>\n",
       "      <td>2018-02-04 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22bb4d85ac19f25ec081e494857ca94dc84f13c8</td>\n",
       "      <td>2018-02-03 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>644</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>96161eebbbcaaadc0c47b24f4815c16e9f7518ce</td>\n",
       "      <td>2018-02-17 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MS</td>\n",
       "      <td>Kiln</td>\n",
       "      <td>18</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>273ce90afa5e8188e6ff7544f58ac1484611ee8f</td>\n",
       "      <td>2018-02-02 08:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>665</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176451</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-02-13 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176452</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-02-13 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176453</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-02-13 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176454</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-02-13 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176455</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-02-01 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>166434 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d74d5111738daa0acc3caa06e4e99ce488c26d14  2018-02-27 15:20:00 UTC   \n",
       "1       ad506c1a54f94de229685985f025564ac449d6a1  2018-02-04 15:00:00 UTC   \n",
       "2       22bb4d85ac19f25ec081e494857ca94dc84f13c8  2018-02-03 14:05:00 UTC   \n",
       "5       96161eebbbcaaadc0c47b24f4815c16e9f7518ce  2018-02-17 17:25:00 UTC   \n",
       "6       273ce90afa5e8188e6ff7544f58ac1484611ee8f  2018-02-02 08:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "176451  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-02-13 19:25:00 UTC   \n",
       "176452  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-02-13 15:05:00 UTC   \n",
       "176453  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-02-13 17:15:00 UTC   \n",
       "176454  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-02-13 16:15:00 UTC   \n",
       "176455  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-02-01 15:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               704                      778   \n",
       "1          heat          hold               721                      718   \n",
       "2          heat          auto               644                      800   \n",
       "5          auto          auto               688                      685   \n",
       "6          heat          hold               665                      666   \n",
       "...         ...           ...               ...                      ...   \n",
       "176451     heat          auto               764                      760   \n",
       "176452     heat          auto               759                      760   \n",
       "176453     heat          auto               759                      760   \n",
       "176454     heat          auto               758                      760   \n",
       "176455     heat          auto               765                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           646            MS   Vancleave   \n",
       "1                           718            MS     Madison   \n",
       "2                           800            MS      Tupelo   \n",
       "5                           635            MS        Kiln   \n",
       "6                           666            MS     Madison   \n",
       "...                         ...           ...         ...   \n",
       "176451                      760            MS  Mooreville   \n",
       "176452                      760            MS  Mooreville   \n",
       "176453                      760            MS  Mooreville   \n",
       "176454                      760            MS  Mooreville   \n",
       "176455                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         8             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "5                        18              True         False             True   \n",
       "6                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "176451                   10             False         False            False   \n",
       "176452                   10             False         False            False   \n",
       "176453                   10             False         False            False   \n",
       "176454                   10             False         False            False   \n",
       "176455                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "5                       Electric  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "176451                       Gas  \n",
       "176452                       Gas  \n",
       "176453                       Gas  \n",
       "176454                       Gas  \n",
       "176455                       Gas  \n",
       "\n",
       "[166434 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/MS/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/MS-day/2019-feb-day-MS.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-02-03 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>742</td>\n",
       "      <td>652</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-02-23 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>675</td>\n",
       "      <td>625</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-02-02 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>742</td>\n",
       "      <td>652</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-02-03 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>742</td>\n",
       "      <td>652</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2019-02-03 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>742</td>\n",
       "      <td>652</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192407</th>\n",
       "      <td>e9457c34b2cd92265d34d254523d8cbc1f11f06c</td>\n",
       "      <td>2019-02-19 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Crystal Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192408</th>\n",
       "      <td>e9457c34b2cd92265d34d254523d8cbc1f11f06c</td>\n",
       "      <td>2019-02-15 13:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Crystal Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192409</th>\n",
       "      <td>e9457c34b2cd92265d34d254523d8cbc1f11f06c</td>\n",
       "      <td>2019-02-15 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Crystal Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192410</th>\n",
       "      <td>e9457c34b2cd92265d34d254523d8cbc1f11f06c</td>\n",
       "      <td>2019-02-17 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Crystal Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192411</th>\n",
       "      <td>e9457c34b2cd92265d34d254523d8cbc1f11f06c</td>\n",
       "      <td>2019-02-16 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>695</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Crystal Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180523 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d23049697dc10f7bda2995dcd45a259452e331c5  2019-02-03 14:40:00 UTC   \n",
       "1       d23049697dc10f7bda2995dcd45a259452e331c5  2019-02-23 18:20:00 UTC   \n",
       "2       d23049697dc10f7bda2995dcd45a259452e331c5  2019-02-02 17:45:00 UTC   \n",
       "3       d23049697dc10f7bda2995dcd45a259452e331c5  2019-02-03 12:55:00 UTC   \n",
       "4       d23049697dc10f7bda2995dcd45a259452e331c5  2019-02-03 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "192407  e9457c34b2cd92265d34d254523d8cbc1f11f06c  2019-02-19 13:05:00 UTC   \n",
       "192408  e9457c34b2cd92265d34d254523d8cbc1f11f06c  2019-02-15 13:00:00 UTC   \n",
       "192409  e9457c34b2cd92265d34d254523d8cbc1f11f06c  2019-02-15 14:05:00 UTC   \n",
       "192410  e9457c34b2cd92265d34d254523d8cbc1f11f06c  2019-02-17 16:45:00 UTC   \n",
       "192411  e9457c34b2cd92265d34d254523d8cbc1f11f06c  2019-02-16 16:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               677                      742   \n",
       "1          auto          hold               668                      675   \n",
       "2          auto          hold               670                      742   \n",
       "3          auto          hold               658                      742   \n",
       "4          auto          hold               664                      742   \n",
       "...         ...           ...               ...                      ...   \n",
       "192407     heat          auto               698                      770   \n",
       "192408     heat          auto               706                      770   \n",
       "192409     heat          auto               706                      770   \n",
       "192410     heat          auto               697                      770   \n",
       "192411     heat          auto               695                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                           652            MS           Belden   \n",
       "1                           625            MS           Belden   \n",
       "2                           652            MS           Belden   \n",
       "3                           652            MS           Belden   \n",
       "4                           652            MS           Belden   \n",
       "...                         ...           ...              ...   \n",
       "192407                      700            MS  Crystal Springs   \n",
       "192408                      700            MS  Crystal Springs   \n",
       "192409                      700            MS  Crystal Springs   \n",
       "192410                      700            MS  Crystal Springs   \n",
       "192411                      700            MS  Crystal Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        20             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "192407                   45             False         False            False   \n",
       "192408                   45             False         False            False   \n",
       "192409                   45             False         False            False   \n",
       "192410                   45             False         False            False   \n",
       "192411                   45             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "192407                       Gas  \n",
       "192408                       Gas  \n",
       "192409                       Gas  \n",
       "192410                       Gas  \n",
       "192411                       Gas  \n",
       "\n",
       "[180523 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/MS/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/MS-day/2020-feb-day-MS.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>18b69a558a6686ecef17b11bebcfa1a6c2379817</td>\n",
       "      <td>2020-02-06 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>752</td>\n",
       "      <td>702</td>\n",
       "      <td>MS</td>\n",
       "      <td>Amory</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2020-02-08 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2020-02-06 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>18b69a558a6686ecef17b11bebcfa1a6c2379817</td>\n",
       "      <td>2020-02-10 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>752</td>\n",
       "      <td>702</td>\n",
       "      <td>MS</td>\n",
       "      <td>Amory</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2020-02-05 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285438</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-02-20 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285439</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-02-20 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285440</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-02-20 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285441</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-02-07 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285442</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2020-02-20 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>280429 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       18b69a558a6686ecef17b11bebcfa1a6c2379817  2020-02-06 18:30:00 UTC   \n",
       "1       f918d405d012b701c0cb10dd807539f0a731ceea  2020-02-08 14:45:00 UTC   \n",
       "3       f918d405d012b701c0cb10dd807539f0a731ceea  2020-02-06 19:30:00 UTC   \n",
       "4       18b69a558a6686ecef17b11bebcfa1a6c2379817  2020-02-10 17:55:00 UTC   \n",
       "5       f918d405d012b701c0cb10dd807539f0a731ceea  2020-02-05 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "285438  748c069371486cb31498d4fa5bec2a5949d72e83  2020-02-20 17:10:00 UTC   \n",
       "285439  748c069371486cb31498d4fa5bec2a5949d72e83  2020-02-20 17:30:00 UTC   \n",
       "285440  748c069371486cb31498d4fa5bec2a5949d72e83  2020-02-20 17:25:00 UTC   \n",
       "285441  748c069371486cb31498d4fa5bec2a5949d72e83  2020-02-07 17:00:00 UTC   \n",
       "285442  748c069371486cb31498d4fa5bec2a5949d72e83  2020-02-20 17:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               687                      752   \n",
       "1          heat          auto               715                      743   \n",
       "3          heat          auto               711                      743   \n",
       "4          auto          hold               703                      752   \n",
       "5          heat          auto               716                      743   \n",
       "...         ...           ...               ...                      ...   \n",
       "285438     auto          hold               742                      800   \n",
       "285439     auto          hold               745                      800   \n",
       "285440     auto          hold               742                      800   \n",
       "285441     auto          hold               709                      800   \n",
       "285442     auto          hold               743                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           702            MS           Amory   \n",
       "1                           717            MS  Pass Christian   \n",
       "3                           717            MS  Pass Christian   \n",
       "4                           702            MS           Amory   \n",
       "5                           717            MS  Pass Christian   \n",
       "...                         ...           ...             ...   \n",
       "285438                      750            MS   Ocean Springs   \n",
       "285439                      750            MS   Ocean Springs   \n",
       "285440                      750            MS   Ocean Springs   \n",
       "285441                      750            MS   Ocean Springs   \n",
       "285442                      750            MS   Ocean Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10              True         False             True   \n",
       "1                        10             False         False             True   \n",
       "3                        10             False         False             True   \n",
       "4                        10              True         False             True   \n",
       "5                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "285438                   19             False         False            False   \n",
       "285439                   19             False         False            False   \n",
       "285440                   19             False         False            False   \n",
       "285441                   19             False         False            False   \n",
       "285442                   19             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "285438                       Gas  \n",
       "285439                       Gas  \n",
       "285440                       Gas  \n",
       "285441                       Gas  \n",
       "285442                       Gas  \n",
       "\n",
       "[280429 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/MS/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/MS-day/2021-feb-day-MS.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8eb3b0d0f522fe3eaa94e40ced1bd19de9792e8c</td>\n",
       "      <td>2021-02-12 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>732</td>\n",
       "      <td>682</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2c6ba90605f8141dbf6c975dd5852d70d875f842</td>\n",
       "      <td>2021-02-18 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>696</td>\n",
       "      <td>696</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2c6ba90605f8141dbf6c975dd5852d70d875f842</td>\n",
       "      <td>2021-02-18 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>696</td>\n",
       "      <td>696</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bc23f35349d4bb4d810c89fa75667f22b5170e4f</td>\n",
       "      <td>2021-02-26 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>79e8f5fa36fa0cb86d9cc37149ddcc769e701c59</td>\n",
       "      <td>2021-02-04 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>779</td>\n",
       "      <td>729</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184996</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2021-02-08 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184997</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2021-02-08 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184998</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2021-02-08 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184999</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2021-02-08 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185000</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2021-02-08 09:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>181993 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8eb3b0d0f522fe3eaa94e40ced1bd19de9792e8c  2021-02-12 13:55:00 UTC   \n",
       "1       2c6ba90605f8141dbf6c975dd5852d70d875f842  2021-02-18 14:15:00 UTC   \n",
       "2       2c6ba90605f8141dbf6c975dd5852d70d875f842  2021-02-18 15:35:00 UTC   \n",
       "3       bc23f35349d4bb4d810c89fa75667f22b5170e4f  2021-02-26 15:40:00 UTC   \n",
       "4       79e8f5fa36fa0cb86d9cc37149ddcc769e701c59  2021-02-04 19:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "184996  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2021-02-08 19:15:00 UTC   \n",
       "184997  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2021-02-08 14:20:00 UTC   \n",
       "184998  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2021-02-08 18:30:00 UTC   \n",
       "184999  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2021-02-08 14:30:00 UTC   \n",
       "185000  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2021-02-08 09:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               678                      732   \n",
       "1          heat          hold               691                      696   \n",
       "2          heat          hold               699                      696   \n",
       "3          cool          hold               668                      666   \n",
       "4          auto          hold               724                      779   \n",
       "...         ...           ...               ...                      ...   \n",
       "184996     heat          hold               764                      760   \n",
       "184997     heat          hold               747                      760   \n",
       "184998     heat          hold               754                      760   \n",
       "184999     heat          hold               762                      760   \n",
       "185000     heat          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           682            MS     Madison   \n",
       "1                           696            MS      Tupelo   \n",
       "2                           696            MS      Tupelo   \n",
       "3                           666            MS     Madison   \n",
       "4                           729            MS     Brandon   \n",
       "...                         ...           ...         ...   \n",
       "184996                      760            MS  Mooreville   \n",
       "184997                      760            MS  Mooreville   \n",
       "184998                      760            MS  Mooreville   \n",
       "184999                      760            MS  Mooreville   \n",
       "185000                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                         0              True         False            False   \n",
       "2                         0              True         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "184996                   10             False         False            False   \n",
       "184997                   10             False         False            False   \n",
       "184998                   10             False         False            False   \n",
       "184999                   10             False         False            False   \n",
       "185000                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "184996                       Gas  \n",
       "184997                       Gas  \n",
       "184998                       Gas  \n",
       "184999                       Gas  \n",
       "185000                       Gas  \n",
       "\n",
       "[181993 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/MS/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>750.444444</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>724.678068</td>\n",
       "      <td>735.730382</td>\n",
       "      <td>735.730382</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>706.048000</td>\n",
       "      <td>710.456000</td>\n",
       "      <td>709.464000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>726.640401</td>\n",
       "      <td>711.283668</td>\n",
       "      <td>711.283668</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>024d1624a424d57f4cdc19a5fa9aa6d90739ef33</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>702.869565</td>\n",
       "      <td>761.369565</td>\n",
       "      <td>711.206522</td>\n",
       "      <td>26.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>304</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>713.960854</td>\n",
       "      <td>711.400356</td>\n",
       "      <td>709.423488</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>643.541667</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>634.937500</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>786.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>741.522026</td>\n",
       "      <td>742.030348</td>\n",
       "      <td>742.030348</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1891 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   feb  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   feb  2017     cool   \n",
       "2    01194744830da7bbc9603d51735c58faef66f72e   feb  2017     heat   \n",
       "3    01194744830da7bbc9603d51735c58faef66f72e   feb  2017     heat   \n",
       "4    024d1624a424d57f4cdc19a5fa9aa6d90739ef33   feb  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "304  fe9bc276b524605ee152e4051285e4e080e77a3f   feb  2021     heat   \n",
       "305  ff5551a641dc7446df1914428ded425dde839080   feb  2021     cool   \n",
       "306  ff5551a641dc7446df1914428ded425dde839080   feb  2021     heat   \n",
       "307  ffe938566049e7ad2eb8c1726ec2215bfda979d7   feb  2021     cool   \n",
       "308  ffe938566049e7ad2eb8c1726ec2215bfda979d7   feb  2021     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            750.444444   \n",
       "1            hold  Ocean Springs            724.678068   \n",
       "2            auto  Ocean Springs            706.048000   \n",
       "3            hold  Ocean Springs            726.640401   \n",
       "4            auto      Horn Lake            702.869565   \n",
       "..            ...            ...                   ...   \n",
       "304          hold         Tupelo            713.960854   \n",
       "305          hold        Madison            643.541667   \n",
       "306          hold        Madison            634.937500   \n",
       "307          hold     Mooreville            786.000000   \n",
       "308          hold     Mooreville            741.522026   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     760.000000                   740.000000   \n",
       "1                     735.730382                   735.730382   \n",
       "2                     710.456000                   709.464000   \n",
       "3                     711.283668                   711.283668   \n",
       "4                     761.369565                   711.206522   \n",
       "..                           ...                          ...   \n",
       "304                   711.400356                   709.423488   \n",
       "305                   690.000000                   690.000000   \n",
       "306                   650.000000                   640.000000   \n",
       "307                   740.000000                   740.000000   \n",
       "308                   742.030348                   742.030348   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                    0.0             False         False             True  \n",
       "3                    0.0             False         False             True  \n",
       "4                   26.0              True         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "304                 75.0             False         False            False  \n",
       "305                 30.0             False         False            False  \n",
       "306                 30.0             False         False            False  \n",
       "307                 10.0             False         False            False  \n",
       "308                 10.0             False         False            False  \n",
       "\n",
       "[1891 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/feb/\" + file)\n",
    "    MS_feb = pd.concat([MS_feb, df])\n",
    "    \n",
    "MS_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_feb.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/MS-day/2017-jun-day-MS.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>77045d611c3cf60d028145a458d955bd64f29192</td>\n",
       "      <td>2017-06-28 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a8324dbb55ef5e62f81ebac499a34cad487b2be0</td>\n",
       "      <td>2017-06-10 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gautier</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22bb4d85ac19f25ec081e494857ca94dc84f13c8</td>\n",
       "      <td>2017-06-25 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>670</td>\n",
       "      <td>800</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2185b8c26725cbbe15a37357dfd1e210a118ff9a</td>\n",
       "      <td>2017-06-03 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>656</td>\n",
       "      <td>MS</td>\n",
       "      <td>Byhalia</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>22bb4d85ac19f25ec081e494857ca94dc84f13c8</td>\n",
       "      <td>2017-06-25 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>670</td>\n",
       "      <td>800</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94626</th>\n",
       "      <td>be35204453ce52fb60900381376b731c0192106b</td>\n",
       "      <td>2017-06-21 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94627</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-06-18 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>793</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94628</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-06-18 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>784</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94629</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-06-18 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>794</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94630</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-06-09 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>91300 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      77045d611c3cf60d028145a458d955bd64f29192  2017-06-28 11:35:00 UTC   \n",
       "1      a8324dbb55ef5e62f81ebac499a34cad487b2be0  2017-06-10 11:35:00 UTC   \n",
       "2      22bb4d85ac19f25ec081e494857ca94dc84f13c8  2017-06-25 19:55:00 UTC   \n",
       "3      2185b8c26725cbbe15a37357dfd1e210a118ff9a  2017-06-03 16:15:00 UTC   \n",
       "6      22bb4d85ac19f25ec081e494857ca94dc84f13c8  2017-06-25 16:50:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "94626  be35204453ce52fb60900381376b731c0192106b  2017-06-21 13:10:00 UTC   \n",
       "94627  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-06-18 15:05:00 UTC   \n",
       "94628  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-06-18 13:20:00 UTC   \n",
       "94629  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-06-18 11:50:00 UTC   \n",
       "94630  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-06-09 12:30:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               734                      735   \n",
       "1         cool          hold               718                      719   \n",
       "2         cool          auto               675                      670   \n",
       "3         auto          hold               718                      718   \n",
       "6         cool          auto               675                      670   \n",
       "...        ...           ...               ...                      ...   \n",
       "94626     cool          hold               759                      760   \n",
       "94627     cool          auto               793                      790   \n",
       "94628     cool          auto               784                      790   \n",
       "94629     cool          auto               794                      790   \n",
       "94630     cool          hold               734                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState        City  Age_of_Home__years_  \\\n",
       "0                          735            MS      Biloxi                   15   \n",
       "1                          719            MS     Gautier                    0   \n",
       "2                          800            MS      Tupelo                   10   \n",
       "3                          656            MS     Byhalia                   15   \n",
       "6                          800            MS      Tupelo                   10   \n",
       "...                        ...           ...         ...                  ...   \n",
       "94626                      760            MS    Meridian                    0   \n",
       "94627                      760            MS     Madison                   10   \n",
       "94628                      760            MS     Madison                   10   \n",
       "94629                      760            MS     Madison                   10   \n",
       "94630                      760            MS  Mooreville                   10   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "6                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "94626              True         False             True   \n",
       "94627             False         False            False   \n",
       "94628             False         False            False   \n",
       "94629             False         False            False   \n",
       "94630             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "6                           Gas  \n",
       "...                         ...  \n",
       "94626                  Electric  \n",
       "94627                       Gas  \n",
       "94628                       Gas  \n",
       "94629                       Gas  \n",
       "94630                       Gas  \n",
       "\n",
       "[91300 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/MS/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/MS-day/2018-jun-day-MS.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cb0a84fc29ae0cb0bff3a61328f2c40d1a61df53</td>\n",
       "      <td>2018-06-06 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>720</td>\n",
       "      <td>718</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a60a0a3e848c4d4eb8396c7cf700b5c1a9093412</td>\n",
       "      <td>2018-06-07 11:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MS</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>a60a0a3e848c4d4eb8396c7cf700b5c1a9093412</td>\n",
       "      <td>2018-06-18 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MS</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4</td>\n",
       "      <td>2018-06-13 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>723</td>\n",
       "      <td>663</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ea10cf5c507d618900db08565bee4a242fd87134</td>\n",
       "      <td>2018-06-10 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254350</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-06-27 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254351</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-06-06 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254352</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-06-27 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254353</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-06-27 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254354</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-06-27 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>239954 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "4       cb0a84fc29ae0cb0bff3a61328f2c40d1a61df53  2018-06-06 19:20:00 UTC   \n",
       "5       a60a0a3e848c4d4eb8396c7cf700b5c1a9093412  2018-06-07 11:35:00 UTC   \n",
       "6       a60a0a3e848c4d4eb8396c7cf700b5c1a9093412  2018-06-18 15:45:00 UTC   \n",
       "10      1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4  2018-06-13 16:05:00 UTC   \n",
       "11      ea10cf5c507d618900db08565bee4a242fd87134  2018-06-10 18:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "254350  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-06-27 16:50:00 UTC   \n",
       "254351  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-06-06 19:55:00 UTC   \n",
       "254352  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-06-27 18:05:00 UTC   \n",
       "254353  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-06-27 17:00:00 UTC   \n",
       "254354  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-06-27 16:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "4          cool          hold               740                      720   \n",
       "5          auto          hold               688                      685   \n",
       "6          auto          hold               681                      685   \n",
       "10         auto          hold               725                      723   \n",
       "11         cool          hold               775                      775   \n",
       "...         ...           ...               ...                      ...   \n",
       "254350     cool          hold               762                      760   \n",
       "254351     cool          hold               763                      760   \n",
       "254352     cool          hold               767                      760   \n",
       "254353     cool          hold               766                      760   \n",
       "254354     cool          hold               761                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "4                           718            MS      Tupelo   \n",
       "5                           635            MS      Newton   \n",
       "6                           635            MS      Newton   \n",
       "10                          663            MS      Tupelo   \n",
       "11                          775            MS     Madison   \n",
       "...                         ...           ...         ...   \n",
       "254350                      760            MS  Mooreville   \n",
       "254351                      760            MS  Mooreville   \n",
       "254352                      760            MS  Mooreville   \n",
       "254353                      760            MS  Mooreville   \n",
       "254354                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "4                        20              True         False            False   \n",
       "5                         0             False         False             True   \n",
       "6                         0             False         False             True   \n",
       "10                       20             False         False            False   \n",
       "11                       10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "254350                   10             False         False            False   \n",
       "254351                   10             False         False            False   \n",
       "254352                   10             False         False            False   \n",
       "254353                   10             False         False            False   \n",
       "254354                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "4                            Gas  \n",
       "5                       Electric  \n",
       "6                       Electric  \n",
       "10                           Gas  \n",
       "11                           Gas  \n",
       "...                          ...  \n",
       "254350                       Gas  \n",
       "254351                       Gas  \n",
       "254352                       Gas  \n",
       "254353                       Gas  \n",
       "254354                       Gas  \n",
       "\n",
       "[239954 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/MS/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/MS-day/2019-jun-day-MS.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6958304af4064a7f1e351b21c0ee3d85f30d02cb</td>\n",
       "      <td>2019-06-07 13:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>670</td>\n",
       "      <td>703</td>\n",
       "      <td>673</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eaee34a616466326eadd77bee921f8d969137b7d</td>\n",
       "      <td>2019-06-07 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>703</td>\n",
       "      <td>653</td>\n",
       "      <td>MS</td>\n",
       "      <td>Southaven</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b97e267eaef40bd27c8fff6a536a0deb3ea69821</td>\n",
       "      <td>2019-06-24 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>685</td>\n",
       "      <td>680</td>\n",
       "      <td>669</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a599456ea9be8ddee6ee185ef3f0e7527c656bf5</td>\n",
       "      <td>2019-06-23 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>MS</td>\n",
       "      <td>Moss Point</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>eaee34a616466326eadd77bee921f8d969137b7d</td>\n",
       "      <td>2019-06-03 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>703</td>\n",
       "      <td>653</td>\n",
       "      <td>MS</td>\n",
       "      <td>Southaven</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348838</th>\n",
       "      <td>e59b56ae8d1b4b366f01f1ea46606f2c366a775d</td>\n",
       "      <td>2019-06-22 09:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Starkville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348839</th>\n",
       "      <td>e59b56ae8d1b4b366f01f1ea46606f2c366a775d</td>\n",
       "      <td>2019-06-22 07:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Starkville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348840</th>\n",
       "      <td>efcfa7e140b4d02ab30082b745226397c0bf4895</td>\n",
       "      <td>2019-06-15 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348841</th>\n",
       "      <td>efcfa7e140b4d02ab30082b745226397c0bf4895</td>\n",
       "      <td>2019-06-13 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348842</th>\n",
       "      <td>efcfa7e140b4d02ab30082b745226397c0bf4895</td>\n",
       "      <td>2019-06-15 11:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>326667 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6958304af4064a7f1e351b21c0ee3d85f30d02cb  2019-06-07 13:05:00 UTC   \n",
       "2       eaee34a616466326eadd77bee921f8d969137b7d  2019-06-07 18:30:00 UTC   \n",
       "3       b97e267eaef40bd27c8fff6a536a0deb3ea69821  2019-06-24 18:10:00 UTC   \n",
       "4       a599456ea9be8ddee6ee185ef3f0e7527c656bf5  2019-06-23 18:35:00 UTC   \n",
       "6       eaee34a616466326eadd77bee921f8d969137b7d  2019-06-03 19:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "348838  e59b56ae8d1b4b366f01f1ea46606f2c366a775d  2019-06-22 09:20:00 UTC   \n",
       "348839  e59b56ae8d1b4b366f01f1ea46606f2c366a775d  2019-06-22 07:30:00 UTC   \n",
       "348840  efcfa7e140b4d02ab30082b745226397c0bf4895  2019-06-15 11:45:00 UTC   \n",
       "348841  efcfa7e140b4d02ab30082b745226397c0bf4895  2019-06-13 12:25:00 UTC   \n",
       "348842  efcfa7e140b4d02ab30082b745226397c0bf4895  2019-06-15 11:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               670                      703   \n",
       "2          auto          auto               703                      703   \n",
       "3          cool          auto               685                      680   \n",
       "4          cool          auto               723                      707   \n",
       "6          auto          auto               720                      703   \n",
       "...         ...           ...               ...                      ...   \n",
       "348838     cool          auto               701                      700   \n",
       "348839     cool          auto               700                      700   \n",
       "348840     cool          auto               703                      700   \n",
       "348841     cool          auto               704                      700   \n",
       "348842     cool          auto               702                      700   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           673            MS      Oxford   \n",
       "2                           653            MS   Southaven   \n",
       "3                           669            MS      Purvis   \n",
       "4                           707            MS  Moss Point   \n",
       "6                           653            MS   Southaven   \n",
       "...                         ...           ...         ...   \n",
       "348838                      700            MS  Starkville   \n",
       "348839                      700            MS  Starkville   \n",
       "348840                      700            MS   Horn Lake   \n",
       "348841                      700            MS   Horn Lake   \n",
       "348842                      700            MS   Horn Lake   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "2                        20              True         False            False   \n",
       "3                        10              True         False             True   \n",
       "4                         9             False         False            False   \n",
       "6                        20              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "348838                   10             False         False            False   \n",
       "348839                   10             False         False            False   \n",
       "348840                   30             False         False            False   \n",
       "348841                   30             False         False            False   \n",
       "348842                   30             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "348838                       Gas  \n",
       "348839                       Gas  \n",
       "348840                       Gas  \n",
       "348841                       Gas  \n",
       "348842                       Gas  \n",
       "\n",
       "[326667 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/MS/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/MS-day/2020-jun-day-MS.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>98903dec3dd9e9c36b342c55aac6231d1f256211</td>\n",
       "      <td>2020-06-20 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>716</td>\n",
       "      <td>684</td>\n",
       "      <td>MS</td>\n",
       "      <td>Olive Branch</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3e4287b86d0da46f17303aedd61e8f5399e30cc3</td>\n",
       "      <td>2020-06-20 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>775</td>\n",
       "      <td>775</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>575c2720bda08f9c686a150357caf94d2a8447e7</td>\n",
       "      <td>2020-06-22 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ovett</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16daf0ca4bfcc11cc2e4721db2c14c38559c0a92</td>\n",
       "      <td>2020-06-28 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>727</td>\n",
       "      <td>727</td>\n",
       "      <td>MS</td>\n",
       "      <td>Amory</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1621d17dd769a418287ee8a1702e662d6d7fe4a1</td>\n",
       "      <td>2020-06-13 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>685</td>\n",
       "      <td>625</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332072</th>\n",
       "      <td>641bbf7373a2e07e201cf1f4fd3c5e6abd37de54</td>\n",
       "      <td>2020-06-04 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>695</td>\n",
       "      <td>665</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332073</th>\n",
       "      <td>641bbf7373a2e07e201cf1f4fd3c5e6abd37de54</td>\n",
       "      <td>2020-06-04 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>695</td>\n",
       "      <td>665</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332074</th>\n",
       "      <td>641bbf7373a2e07e201cf1f4fd3c5e6abd37de54</td>\n",
       "      <td>2020-06-23 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>695</td>\n",
       "      <td>665</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332075</th>\n",
       "      <td>641bbf7373a2e07e201cf1f4fd3c5e6abd37de54</td>\n",
       "      <td>2020-06-15 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>695</td>\n",
       "      <td>665</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>332076</th>\n",
       "      <td>641bbf7373a2e07e201cf1f4fd3c5e6abd37de54</td>\n",
       "      <td>2020-06-17 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>695</td>\n",
       "      <td>665</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>319030 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       98903dec3dd9e9c36b342c55aac6231d1f256211  2020-06-20 14:10:00 UTC   \n",
       "1       3e4287b86d0da46f17303aedd61e8f5399e30cc3  2020-06-20 14:25:00 UTC   \n",
       "2       575c2720bda08f9c686a150357caf94d2a8447e7  2020-06-22 10:50:00 UTC   \n",
       "3       16daf0ca4bfcc11cc2e4721db2c14c38559c0a92  2020-06-28 19:15:00 UTC   \n",
       "4       1621d17dd769a418287ee8a1702e662d6d7fe4a1  2020-06-13 18:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "332072  641bbf7373a2e07e201cf1f4fd3c5e6abd37de54  2020-06-04 15:35:00 UTC   \n",
       "332073  641bbf7373a2e07e201cf1f4fd3c5e6abd37de54  2020-06-04 19:50:00 UTC   \n",
       "332074  641bbf7373a2e07e201cf1f4fd3c5e6abd37de54  2020-06-23 18:40:00 UTC   \n",
       "332075  641bbf7373a2e07e201cf1f4fd3c5e6abd37de54  2020-06-15 14:30:00 UTC   \n",
       "332076  641bbf7373a2e07e201cf1f4fd3c5e6abd37de54  2020-06-17 18:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               702                      716   \n",
       "1          cool          hold               751                      775   \n",
       "2          cool          hold               700                      718   \n",
       "3          cool          hold               731                      727   \n",
       "4          cool          hold               687                      685   \n",
       "...         ...           ...               ...                      ...   \n",
       "332072     auto          hold               693                      695   \n",
       "332073     auto          hold               686                      695   \n",
       "332074     auto          hold               701                      695   \n",
       "332075     auto          hold               701                      695   \n",
       "332076     auto          hold               693                      695   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           684            MS  Olive Branch   \n",
       "1                           775            MS       Madison   \n",
       "2                           718            MS         Ovett   \n",
       "3                           727            MS         Amory   \n",
       "4                           625            MS        Oxford   \n",
       "...                         ...           ...           ...   \n",
       "332072                      665            MS        Oxford   \n",
       "332073                      665            MS        Oxford   \n",
       "332074                      665            MS        Oxford   \n",
       "332075                      665            MS        Oxford   \n",
       "332076                      665            MS        Oxford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        19              True         False            False   \n",
       "2                         9             False         False            False   \n",
       "3                        10              True         False             True   \n",
       "4                        30              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "332072                   10             False         False            False   \n",
       "332073                   10             False         False            False   \n",
       "332074                   10             False         False            False   \n",
       "332075                   10             False         False            False   \n",
       "332076                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "332072                       Gas  \n",
       "332073                       Gas  \n",
       "332074                       Gas  \n",
       "332075                       Gas  \n",
       "332076                       Gas  \n",
       "\n",
       "[319030 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/MS/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/MS-day/2021-jun-day-MS.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eb6fcfbb6678f3fbff59c8951af2325210523719</td>\n",
       "      <td>2021-06-17 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>MS</td>\n",
       "      <td>Columbus</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a27e9562d509962fa19524e68a06dec21f2503fb</td>\n",
       "      <td>2021-06-04 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8bb37505af1f12286cb960fed0eb3177c49e121b</td>\n",
       "      <td>2021-06-01 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>702</td>\n",
       "      <td>652</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6f4cf8588e5f3242d3ccd0fd933c7f4b6764c5b1</td>\n",
       "      <td>2021-06-06 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Moss Point</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>eb6fcfbb6678f3fbff59c8951af2325210523719</td>\n",
       "      <td>2021-06-17 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>MS</td>\n",
       "      <td>Columbus</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205399</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-06-05 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205400</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-06-06 09:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205401</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-06-06 09:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205402</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-06-06 07:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>205403</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-06-05 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>199219 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       eb6fcfbb6678f3fbff59c8951af2325210523719  2021-06-17 19:20:00 UTC   \n",
       "1       a27e9562d509962fa19524e68a06dec21f2503fb  2021-06-04 15:20:00 UTC   \n",
       "2       8bb37505af1f12286cb960fed0eb3177c49e121b  2021-06-01 19:00:00 UTC   \n",
       "3       6f4cf8588e5f3242d3ccd0fd933c7f4b6764c5b1  2021-06-06 18:30:00 UTC   \n",
       "4       eb6fcfbb6678f3fbff59c8951af2325210523719  2021-06-17 18:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "205399  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-06-05 14:25:00 UTC   \n",
       "205400  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-06-06 09:50:00 UTC   \n",
       "205401  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-06-06 09:30:00 UTC   \n",
       "205402  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-06-06 07:00:00 UTC   \n",
       "205403  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-06-05 15:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               746                      749   \n",
       "1          cool          hold               717                      722   \n",
       "2          auto          hold               708                      702   \n",
       "3          cool          hold               740                      735   \n",
       "4          cool          hold               751                      749   \n",
       "...         ...           ...               ...                      ...   \n",
       "205399     cool          hold               763                      760   \n",
       "205400     cool          hold               751                      760   \n",
       "205401     cool          hold               754                      760   \n",
       "205402     cool          hold               752                      760   \n",
       "205403     cool          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           749            MS    Columbus   \n",
       "1                           722            MS      Oxford   \n",
       "2                           652            MS     Brandon   \n",
       "3                           735            MS  Moss Point   \n",
       "4                           749            MS    Columbus   \n",
       "...                         ...           ...         ...   \n",
       "205399                      760            MS     Madison   \n",
       "205400                      760            MS     Madison   \n",
       "205401                      760            MS     Madison   \n",
       "205402                      760            MS     Madison   \n",
       "205403                      760            MS     Madison   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        39             False         False             True   \n",
       "1                        30             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                        39             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "205399                   10             False         False            False   \n",
       "205400                   10             False         False            False   \n",
       "205401                   10             False         False            False   \n",
       "205402                   10             False         False            False   \n",
       "205403                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "205399                       Gas  \n",
       "205400                       Gas  \n",
       "205401                       Gas  \n",
       "205402                       Gas  \n",
       "205403                       Gas  \n",
       "\n",
       "[199219 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/MS/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>738.008032</td>\n",
       "      <td>736.922691</td>\n",
       "      <td>676.021084</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>735.143590</td>\n",
       "      <td>736.944872</td>\n",
       "      <td>736.944872</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>022c67a9cc7538740068f7248330a614c0ddb96f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>708.250000</td>\n",
       "      <td>719.375000</td>\n",
       "      <td>669.375000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>022c67a9cc7538740068f7248330a614c0ddb96f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>710.983607</td>\n",
       "      <td>714.721311</td>\n",
       "      <td>684.721311</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>697.459306</td>\n",
       "      <td>698.123142</td>\n",
       "      <td>633.970276</td>\n",
       "      <td>26.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>232</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>731.842051</td>\n",
       "      <td>730.814017</td>\n",
       "      <td>730.814017</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>709.118644</td>\n",
       "      <td>705.403955</td>\n",
       "      <td>705.403955</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>708.410112</td>\n",
       "      <td>716.943820</td>\n",
       "      <td>715.078652</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>733.231047</td>\n",
       "      <td>731.041742</td>\n",
       "      <td>731.041742</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>236</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>741.214286</td>\n",
       "      <td>731.285714</td>\n",
       "      <td>731.285714</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1608 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   jun  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   jun  2017     cool   \n",
       "2    022c67a9cc7538740068f7248330a614c0ddb96f   jun  2017     auto   \n",
       "3    022c67a9cc7538740068f7248330a614c0ddb96f   jun  2017     cool   \n",
       "4    0442aded7dc2237382d8074027fe1db6823268d0   jun  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "232  fdfe0112380a8ee36c3dfe16899298a72c886d49   jun  2021     cool   \n",
       "233  fe9bc276b524605ee152e4051285e4e080e77a3f   jun  2021     cool   \n",
       "234  ff5551a641dc7446df1914428ded425dde839080   jun  2021     cool   \n",
       "235  ffe938566049e7ad2eb8c1726ec2215bfda979d7   jun  2021     cool   \n",
       "236  ffe938566049e7ad2eb8c1726ec2215bfda979d7   jun  2021     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            738.008032   \n",
       "1            hold  Ocean Springs            735.143590   \n",
       "2            auto         Biloxi            708.250000   \n",
       "3            auto         Biloxi            710.983607   \n",
       "4            hold         Oxford            697.459306   \n",
       "..            ...            ...                   ...   \n",
       "232          hold        Jackson            731.842051   \n",
       "233          hold         Tupelo            709.118644   \n",
       "234          hold        Madison            708.410112   \n",
       "235          hold     Mooreville            733.231047   \n",
       "236          hold     Mooreville            741.214286   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     736.922691                   676.021084   \n",
       "1                     736.944872                   736.944872   \n",
       "2                     719.375000                   669.375000   \n",
       "3                     714.721311                   684.721311   \n",
       "4                     698.123142                   633.970276   \n",
       "..                           ...                          ...   \n",
       "232                   730.814017                   730.814017   \n",
       "233                   705.403955                   705.403955   \n",
       "234                   716.943820                   715.078652   \n",
       "235                   731.041742                   731.041742   \n",
       "236                   731.285714                   731.285714   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                    5.0              True         False             True  \n",
       "3                    5.0              True         False             True  \n",
       "4                   26.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "232                 30.0             False         False            False  \n",
       "233                 75.0             False         False            False  \n",
       "234                 30.0             False         False            False  \n",
       "235                 10.0             False         False            False  \n",
       "236                 10.0             False         False            False  \n",
       "\n",
       "[1608 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/jun/\" + file)\n",
    "    MS_jun = pd.concat([MS_jun, df])\n",
    "    \n",
    "MS_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_jun.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/MS-day/2017-jul-day-MS.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53a9f3f8f5ec8a83916689c53046deb93598bb85</td>\n",
       "      <td>2017-07-15 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2017-07-16 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2017-07-19 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>782aa187364048e83a195c76583bda416532d39d</td>\n",
       "      <td>2017-07-19 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>765</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a8324dbb55ef5e62f81ebac499a34cad487b2be0</td>\n",
       "      <td>2017-07-04 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gautier</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103968</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-07-09 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>796</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103969</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-07-09 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>787</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103970</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-07-09 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>776</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103971</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-07-09 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>791</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103972</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2017-07-09 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>786</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>98289 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       53a9f3f8f5ec8a83916689c53046deb93598bb85  2017-07-15 12:25:00 UTC   \n",
       "1       d23049697dc10f7bda2995dcd45a259452e331c5  2017-07-16 17:55:00 UTC   \n",
       "2       d23049697dc10f7bda2995dcd45a259452e331c5  2017-07-19 11:45:00 UTC   \n",
       "3       782aa187364048e83a195c76583bda416532d39d  2017-07-19 15:20:00 UTC   \n",
       "4       a8324dbb55ef5e62f81ebac499a34cad487b2be0  2017-07-04 17:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "103968  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-07-09 19:40:00 UTC   \n",
       "103969  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-07-09 14:45:00 UTC   \n",
       "103970  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-07-09 12:15:00 UTC   \n",
       "103971  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-07-09 17:30:00 UTC   \n",
       "103972  f0674498c898912150ff0fd0cff9c42ea053a96c  2017-07-09 14:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      745   \n",
       "1          cool          hold               739                      735   \n",
       "2          cool          hold               730                      735   \n",
       "3          auto          hold               731                      765   \n",
       "4          cool          hold               723                      718   \n",
       "...         ...           ...               ...                      ...   \n",
       "103968     cool          auto               796                      790   \n",
       "103969     cool          auto               787                      790   \n",
       "103970     cool          auto               776                      790   \n",
       "103971     cool          auto               791                      790   \n",
       "103972     cool          auto               786                      790   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           745            MS  Madison                   15   \n",
       "1                           735            MS   Belden                   20   \n",
       "2                           735            MS   Belden                   20   \n",
       "3                           735            MS  Madison                   25   \n",
       "4                           718            MS  Gautier                    0   \n",
       "...                         ...           ...      ...                  ...   \n",
       "103968                      760            MS  Madison                   10   \n",
       "103969                      760            MS  Madison                   10   \n",
       "103970                      760            MS  Madison                   10   \n",
       "103971                      760            MS  Madison                   10   \n",
       "103972                      760            MS  Madison                   10   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "103968             False         False            False   \n",
       "103969             False         False            False   \n",
       "103970             False         False            False   \n",
       "103971             False         False            False   \n",
       "103972             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "103968                       Gas  \n",
       "103969                       Gas  \n",
       "103970                       Gas  \n",
       "103971                       Gas  \n",
       "103972                       Gas  \n",
       "\n",
       "[98289 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/MS/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/MS-day/2018-jul-day-MS.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b5173564363b407ac06c9701134b97a7c42f3d03</td>\n",
       "      <td>2018-07-13 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>48</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d2da7494cf18bad8fe144eccffcce8d1ff3d9817</td>\n",
       "      <td>2018-07-18 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MS</td>\n",
       "      <td>Flowood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>79e8f5fa36fa0cb86d9cc37149ddcc769e701c59</td>\n",
       "      <td>2018-07-18 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2018-07-22 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d2da7494cf18bad8fe144eccffcce8d1ff3d9817</td>\n",
       "      <td>2018-07-17 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>MS</td>\n",
       "      <td>Flowood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264915</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-07-27 10:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264916</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-07-25 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264917</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-07-27 09:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264918</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-07-27 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264919</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-07-20 10:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>252111 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b5173564363b407ac06c9701134b97a7c42f3d03  2018-07-13 19:30:00 UTC   \n",
       "2       d2da7494cf18bad8fe144eccffcce8d1ff3d9817  2018-07-18 11:50:00 UTC   \n",
       "3       79e8f5fa36fa0cb86d9cc37149ddcc769e701c59  2018-07-18 18:30:00 UTC   \n",
       "4       f918d405d012b701c0cb10dd807539f0a731ceea  2018-07-22 15:25:00 UTC   \n",
       "5       d2da7494cf18bad8fe144eccffcce8d1ff3d9817  2018-07-17 14:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "264915  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-07-27 10:10:00 UTC   \n",
       "264916  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-07-25 13:55:00 UTC   \n",
       "264917  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-07-27 09:20:00 UTC   \n",
       "264918  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-07-27 10:40:00 UTC   \n",
       "264919  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-07-20 10:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      692   \n",
       "2          cool          hold               743                      745   \n",
       "3          cool          hold               750                      752   \n",
       "4          cool          hold               779                      762   \n",
       "5          cool          hold               754                      765   \n",
       "...         ...           ...               ...                      ...   \n",
       "264915     cool          auto               751                      760   \n",
       "264916     cool          auto               763                      760   \n",
       "264917     cool          auto               765                      760   \n",
       "264918     cool          auto               764                      760   \n",
       "264919     cool          auto               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           692            MS          Tupelo   \n",
       "2                           745            MS         Flowood   \n",
       "3                           752            MS         Brandon   \n",
       "4                           762            MS  Pass Christian   \n",
       "5                           765            MS         Flowood   \n",
       "...                         ...           ...             ...   \n",
       "264915                      760            MS      Mooreville   \n",
       "264916                      760            MS      Mooreville   \n",
       "264917                      760            MS      Mooreville   \n",
       "264918                      760            MS      Mooreville   \n",
       "264919                      760            MS      Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        48              True         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        10             False         False             True   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "264915                   10             False         False            False   \n",
       "264916                   10             False         False            False   \n",
       "264917                   10             False         False            False   \n",
       "264918                   10             False         False            False   \n",
       "264919                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "264915                       Gas  \n",
       "264916                       Gas  \n",
       "264917                       Gas  \n",
       "264918                       Gas  \n",
       "264919                       Gas  \n",
       "\n",
       "[252111 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/MS/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/MS-day/2019-jul-day-MS.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b97e267eaef40bd27c8fff6a536a0deb3ea69821</td>\n",
       "      <td>2019-07-22 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>719</td>\n",
       "      <td>679</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4</td>\n",
       "      <td>2019-07-29 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>703</td>\n",
       "      <td>653</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>eaee34a616466326eadd77bee921f8d969137b7d</td>\n",
       "      <td>2019-07-12 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>693</td>\n",
       "      <td>643</td>\n",
       "      <td>MS</td>\n",
       "      <td>Southaven</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3c67e64a200d91a6a19384106382ec07affa3475</td>\n",
       "      <td>2019-07-15 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>717</td>\n",
       "      <td>667</td>\n",
       "      <td>MS</td>\n",
       "      <td>Starkville</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>e61f7de1790ed6e52ac20dcde3492c8b9d485a1a</td>\n",
       "      <td>2019-07-14 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>673</td>\n",
       "      <td>MS</td>\n",
       "      <td>Long Beach</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358022</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2019-07-14 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358023</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2019-07-14 07:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358024</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2019-07-14 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358025</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2019-07-14 08:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358026</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2019-07-14 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>338548 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "3       b97e267eaef40bd27c8fff6a536a0deb3ea69821  2019-07-22 15:15:00 UTC   \n",
       "4       1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4  2019-07-29 18:25:00 UTC   \n",
       "6       eaee34a616466326eadd77bee921f8d969137b7d  2019-07-12 18:45:00 UTC   \n",
       "7       3c67e64a200d91a6a19384106382ec07affa3475  2019-07-15 19:50:00 UTC   \n",
       "10      e61f7de1790ed6e52ac20dcde3492c8b9d485a1a  2019-07-14 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "358022  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2019-07-14 16:30:00 UTC   \n",
       "358023  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2019-07-14 07:10:00 UTC   \n",
       "358024  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2019-07-14 16:05:00 UTC   \n",
       "358025  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2019-07-14 08:40:00 UTC   \n",
       "358026  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2019-07-14 13:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "3          cool          auto               711                      719   \n",
       "4          auto          hold               727                      703   \n",
       "6          auto          auto               696                      693   \n",
       "7          auto          hold               730                      717   \n",
       "10         auto          hold               730                      730   \n",
       "...         ...           ...               ...                      ...   \n",
       "358022     cool          auto               765                      760   \n",
       "358023     cool          auto               764                      760   \n",
       "358024     cool          auto               763                      760   \n",
       "358025     cool          auto               766                      760   \n",
       "358026     cool          auto               756                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "3                           679            MS      Purvis   \n",
       "4                           653            MS      Tupelo   \n",
       "6                           643            MS   Southaven   \n",
       "7                           667            MS  Starkville   \n",
       "10                          673            MS  Long Beach   \n",
       "...                         ...           ...         ...   \n",
       "358022                      760            MS  Mooreville   \n",
       "358023                      760            MS  Mooreville   \n",
       "358024                      760            MS  Mooreville   \n",
       "358025                      760            MS  Mooreville   \n",
       "358026                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "3                        10              True         False             True   \n",
       "4                        20             False         False            False   \n",
       "6                        20              True         False            False   \n",
       "7                         5             False         False            False   \n",
       "10                       40             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "358022                   10             False         False            False   \n",
       "358023                   10             False         False            False   \n",
       "358024                   10             False         False            False   \n",
       "358025                   10             False         False            False   \n",
       "358026                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "7                            Gas  \n",
       "10                           Gas  \n",
       "...                          ...  \n",
       "358022                       Gas  \n",
       "358023                       Gas  \n",
       "358024                       Gas  \n",
       "358025                       Gas  \n",
       "358026                       Gas  \n",
       "\n",
       "[338548 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/MS/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/MS-day/2020-jul-day-MS.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3</td>\n",
       "      <td>2020-07-06 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gulfport</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3</td>\n",
       "      <td>2020-07-02 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gulfport</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eb6fcfbb6678f3fbff59c8951af2325210523719</td>\n",
       "      <td>2020-07-27 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>739</td>\n",
       "      <td>739</td>\n",
       "      <td>MS</td>\n",
       "      <td>Columbus</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b97e267eaef40bd27c8fff6a536a0deb3ea69821</td>\n",
       "      <td>2020-07-21 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d24c955854398897407bb1cc7e8470981db392c1</td>\n",
       "      <td>2020-07-26 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>697</td>\n",
       "      <td>647</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327954</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>2020-07-02 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327955</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>2020-07-01 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327956</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>2020-07-01 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327957</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>2020-07-16 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327958</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>2020-07-16 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>307892 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3  2020-07-06 12:30:00 UTC   \n",
       "1       e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3  2020-07-02 13:40:00 UTC   \n",
       "2       eb6fcfbb6678f3fbff59c8951af2325210523719  2020-07-27 12:15:00 UTC   \n",
       "3       b97e267eaef40bd27c8fff6a536a0deb3ea69821  2020-07-21 13:35:00 UTC   \n",
       "5       d24c955854398897407bb1cc7e8470981db392c1  2020-07-26 16:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "327954  fdfe0112380a8ee36c3dfe16899298a72c886d49  2020-07-02 18:40:00 UTC   \n",
       "327955  fdfe0112380a8ee36c3dfe16899298a72c886d49  2020-07-01 19:50:00 UTC   \n",
       "327956  fdfe0112380a8ee36c3dfe16899298a72c886d49  2020-07-01 15:05:00 UTC   \n",
       "327957  fe9bc276b524605ee152e4051285e4e080e77a3f  2020-07-16 19:15:00 UTC   \n",
       "327958  fe9bc276b524605ee152e4051285e4e080e77a3f  2020-07-16 18:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               720                      719   \n",
       "1          cool          hold               723                      719   \n",
       "2          cool          hold               737                      739   \n",
       "3          cool          hold               674                      677   \n",
       "5          auto          hold               698                      697   \n",
       "...         ...           ...               ...                      ...   \n",
       "327954     cool          hold               759                      760   \n",
       "327955     cool          hold               761                      760   \n",
       "327956     cool          hold               761                      760   \n",
       "327957     cool          auto               770                      760   \n",
       "327958     cool          auto               770                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           719            MS  Gulfport                   20   \n",
       "1                           719            MS  Gulfport                   20   \n",
       "2                           739            MS  Columbus                   39   \n",
       "3                           677            MS    Purvis                   10   \n",
       "5                           647            MS    Oxford                    0   \n",
       "...                         ...           ...       ...                  ...   \n",
       "327954                      760            MS   Jackson                   30   \n",
       "327955                      760            MS   Jackson                   30   \n",
       "327956                      760            MS   Jackson                   30   \n",
       "327957                      760            MS    Tupelo                   75   \n",
       "327958                      760            MS    Tupelo                   75   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False             True   \n",
       "1                  False         False             True   \n",
       "2                  False         False             True   \n",
       "3                   True         False             True   \n",
       "5                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "327954             False         False            False   \n",
       "327955             False         False            False   \n",
       "327956             False         False            False   \n",
       "327957             False         False            False   \n",
       "327958             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "327954                       Gas  \n",
       "327955                       Gas  \n",
       "327956                       Gas  \n",
       "327957                       Gas  \n",
       "327958                       Gas  \n",
       "\n",
       "[307892 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/MS/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/MS-day/2021-jul-day-MS.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8d5497d8eeb1d76fa99ce5d78b2fa338d679d805</td>\n",
       "      <td>2021-07-10 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8bb37505af1f12286cb960fed0eb3177c49e121b</td>\n",
       "      <td>2021-07-24 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>686</td>\n",
       "      <td>636</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a27e9562d509962fa19524e68a06dec21f2503fb</td>\n",
       "      <td>2021-07-29 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2bc31afd13f246dba0252b321a1b4e32381e39ad</td>\n",
       "      <td>2021-07-11 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>713</td>\n",
       "      <td>653</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c5b7a9591a03cc5fc6d0be096d68324779eb3088</td>\n",
       "      <td>2021-07-21 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>798</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gulfport</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203024</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-07-01 08:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203025</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-07-22 08:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203026</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-07-18 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203027</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-07-29 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203028</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2021-07-01 08:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>194515 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8d5497d8eeb1d76fa99ce5d78b2fa338d679d805  2021-07-10 16:45:00 UTC   \n",
       "1       8bb37505af1f12286cb960fed0eb3177c49e121b  2021-07-24 12:50:00 UTC   \n",
       "2       a27e9562d509962fa19524e68a06dec21f2503fb  2021-07-29 12:55:00 UTC   \n",
       "3       2bc31afd13f246dba0252b321a1b4e32381e39ad  2021-07-11 19:00:00 UTC   \n",
       "4       c5b7a9591a03cc5fc6d0be096d68324779eb3088  2021-07-21 10:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "203024  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-07-01 08:55:00 UTC   \n",
       "203025  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-07-22 08:55:00 UTC   \n",
       "203026  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-07-18 19:25:00 UTC   \n",
       "203027  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-07-29 14:25:00 UTC   \n",
       "203028  f0674498c898912150ff0fd0cff9c42ea053a96c  2021-07-01 08:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               747                      755   \n",
       "1          auto          hold               697                      686   \n",
       "2          cool          hold               709                      712   \n",
       "3          auto          hold               710                      713   \n",
       "4          cool          hold               798                      800   \n",
       "...         ...           ...               ...                      ...   \n",
       "203024     cool          hold               766                      760   \n",
       "203025     cool          hold               754                      760   \n",
       "203026     cool          hold               768                      760   \n",
       "203027     cool          hold               757                      760   \n",
       "203028     cool          hold               763                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           755            MS  Ocean Springs   \n",
       "1                           636            MS        Brandon   \n",
       "2                           712            MS         Oxford   \n",
       "3                           653            MS        Clinton   \n",
       "4                           800            MS       Gulfport   \n",
       "...                         ...           ...            ...   \n",
       "203024                      760            MS        Madison   \n",
       "203025                      760            MS        Madison   \n",
       "203026                      760            MS        Madison   \n",
       "203027                      760            MS        Madison   \n",
       "203028                      760            MS        Madison   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False             True   \n",
       "1                        10             False         False            False   \n",
       "2                        30             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        55             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "203024                   10             False         False            False   \n",
       "203025                   10             False         False            False   \n",
       "203026                   10             False         False            False   \n",
       "203027                   10             False         False            False   \n",
       "203028                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "203024                       Gas  \n",
       "203025                       Gas  \n",
       "203026                       Gas  \n",
       "203027                       Gas  \n",
       "203028                       Gas  \n",
       "\n",
       "[194515 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/MS/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>738.582902</td>\n",
       "      <td>737.663212</td>\n",
       "      <td>738.264249</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>742.113695</td>\n",
       "      <td>743.492248</td>\n",
       "      <td>743.490956</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>024d1624a424d57f4cdc19a5fa9aa6d90739ef33</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>721.333333</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>689.952381</td>\n",
       "      <td>690.857143</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>698.350642</td>\n",
       "      <td>698.817979</td>\n",
       "      <td>639.062535</td>\n",
       "      <td>26.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>faa211ab8d7004a4f676ea83f1169cda20554042</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Moss Point</td>\n",
       "      <td>690.867577</td>\n",
       "      <td>689.625939</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>730.270657</td>\n",
       "      <td>729.854939</td>\n",
       "      <td>729.854939</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>725.833333</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>217</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>721.467980</td>\n",
       "      <td>724.310345</td>\n",
       "      <td>721.541872</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>729.247783</td>\n",
       "      <td>723.935059</td>\n",
       "      <td>723.935059</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1611 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   jul  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   jul  2017     cool   \n",
       "2    024d1624a424d57f4cdc19a5fa9aa6d90739ef33   jul  2017     auto   \n",
       "3    0442aded7dc2237382d8074027fe1db6823268d0   jul  2017     auto   \n",
       "4    0442aded7dc2237382d8074027fe1db6823268d0   jul  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "214  faa211ab8d7004a4f676ea83f1169cda20554042   jul  2021     auto   \n",
       "215  fdfe0112380a8ee36c3dfe16899298a72c886d49   jul  2021     cool   \n",
       "216  fe9bc276b524605ee152e4051285e4e080e77a3f   jul  2021     cool   \n",
       "217  ff5551a641dc7446df1914428ded425dde839080   jul  2021     cool   \n",
       "218  ffe938566049e7ad2eb8c1726ec2215bfda979d7   jul  2021     cool   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            738.582902   \n",
       "1            hold  Ocean Springs            742.113695   \n",
       "2            hold      Horn Lake            721.333333   \n",
       "3            auto         Oxford            689.952381   \n",
       "4            hold         Oxford            698.350642   \n",
       "..            ...            ...                   ...   \n",
       "214          hold     Moss Point            690.867577   \n",
       "215          hold        Jackson            730.270657   \n",
       "216          hold         Tupelo            725.833333   \n",
       "217          hold        Madison            721.467980   \n",
       "218          hold     Mooreville            729.247783   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     737.663212                   738.264249   \n",
       "1                     743.492248                   743.490956   \n",
       "2                     720.000000                   670.000000   \n",
       "3                     690.857143                   640.000000   \n",
       "4                     698.817979                   639.062535   \n",
       "..                           ...                          ...   \n",
       "214                   689.625939                   630.000000   \n",
       "215                   729.854939                   729.854939   \n",
       "216                   720.000000                   720.000000   \n",
       "217                   724.310345                   721.541872   \n",
       "218                   723.935059                   723.935059   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                   26.0              True         False             True  \n",
       "3                   26.0             False         False            False  \n",
       "4                   26.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "214                 20.0              True         False             True  \n",
       "215                 30.0             False         False            False  \n",
       "216                 75.0             False         False            False  \n",
       "217                 30.0             False         False            False  \n",
       "218                 10.0             False         False            False  \n",
       "\n",
       "[1611 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/jul/\" + file)\n",
    "    MS_jul = pd.concat([MS_jul, df])\n",
    "    \n",
    "MS_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_jul.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/MS-day/2017-aug-day-MS.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a60a0a3e848c4d4eb8396c7cf700b5c1a9093412</td>\n",
       "      <td>2017-08-17 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>771</td>\n",
       "      <td>651</td>\n",
       "      <td>MS</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4fe01e07e4cc830b6998642dd0327a85d13b615a</td>\n",
       "      <td>2017-08-26 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>739</td>\n",
       "      <td>678</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a60a0a3e848c4d4eb8396c7cf700b5c1a9093412</td>\n",
       "      <td>2017-08-30 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>721</td>\n",
       "      <td>651</td>\n",
       "      <td>MS</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>53a9f3f8f5ec8a83916689c53046deb93598bb85</td>\n",
       "      <td>2017-08-11 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a60a0a3e848c4d4eb8396c7cf700b5c1a9093412</td>\n",
       "      <td>2017-08-26 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>751</td>\n",
       "      <td>651</td>\n",
       "      <td>MS</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115369</th>\n",
       "      <td>be35204453ce52fb60900381376b731c0192106b</td>\n",
       "      <td>2017-08-05 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115370</th>\n",
       "      <td>be35204453ce52fb60900381376b731c0192106b</td>\n",
       "      <td>2017-08-07 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115371</th>\n",
       "      <td>be35204453ce52fb60900381376b731c0192106b</td>\n",
       "      <td>2017-08-31 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115372</th>\n",
       "      <td>be35204453ce52fb60900381376b731c0192106b</td>\n",
       "      <td>2017-08-05 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115373</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-08-16 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>745</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>110135 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a60a0a3e848c4d4eb8396c7cf700b5c1a9093412  2017-08-17 19:05:00 UTC   \n",
       "1       4fe01e07e4cc830b6998642dd0327a85d13b615a  2017-08-26 16:20:00 UTC   \n",
       "2       a60a0a3e848c4d4eb8396c7cf700b5c1a9093412  2017-08-30 14:25:00 UTC   \n",
       "3       53a9f3f8f5ec8a83916689c53046deb93598bb85  2017-08-11 12:45:00 UTC   \n",
       "4       a60a0a3e848c4d4eb8396c7cf700b5c1a9093412  2017-08-26 17:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "115369  be35204453ce52fb60900381376b731c0192106b  2017-08-05 16:15:00 UTC   \n",
       "115370  be35204453ce52fb60900381376b731c0192106b  2017-08-07 12:40:00 UTC   \n",
       "115371  be35204453ce52fb60900381376b731c0192106b  2017-08-31 17:40:00 UTC   \n",
       "115372  be35204453ce52fb60900381376b731c0192106b  2017-08-05 16:10:00 UTC   \n",
       "115373  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-08-16 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               769                      771   \n",
       "1          auto          hold               723                      739   \n",
       "2          auto          hold               718                      721   \n",
       "3          cool          hold               690                      735   \n",
       "4          auto          hold               746                      751   \n",
       "...         ...           ...               ...                      ...   \n",
       "115369     cool          hold               763                      760   \n",
       "115370     cool          hold               762                      760   \n",
       "115371     cool          hold               758                      760   \n",
       "115372     cool          hold               762                      760   \n",
       "115373     cool          auto               745                      740   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           651            MS      Newton   \n",
       "1                           678            MS     Jackson   \n",
       "2                           651            MS      Newton   \n",
       "3                           735            MS     Madison   \n",
       "4                           651            MS      Newton   \n",
       "...                         ...           ...         ...   \n",
       "115369                      760            MS    Meridian   \n",
       "115370                      760            MS    Meridian   \n",
       "115371                      760            MS    Meridian   \n",
       "115372                      760            MS    Meridian   \n",
       "115373                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False             True   \n",
       "1                        20             False         False            False   \n",
       "2                         0             False         False             True   \n",
       "3                        15             False         False            False   \n",
       "4                         0             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "115369                    0              True         False             True   \n",
       "115370                    0              True         False             True   \n",
       "115371                    0              True         False             True   \n",
       "115372                    0              True         False             True   \n",
       "115373                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "115369                  Electric  \n",
       "115370                  Electric  \n",
       "115371                  Electric  \n",
       "115372                  Electric  \n",
       "115373                       Gas  \n",
       "\n",
       "[110135 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/MS/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/MS-day/2018-aug-day-MS.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>2018-08-21 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>722</td>\n",
       "      <td>672</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>26</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3</td>\n",
       "      <td>2018-08-13 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gulfport</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3471cc502c2f35682aac256d68cc86083f4bd28f</td>\n",
       "      <td>2018-08-11 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b92895e7b640505e4844b475ebf1d76483786365</td>\n",
       "      <td>2018-08-26 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b92895e7b640505e4844b475ebf1d76483786365</td>\n",
       "      <td>2018-08-25 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258252</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-08-05 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258253</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-08-05 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258254</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-08-05 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258255</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-08-18 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258256</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-08-21 11:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>246362 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0442aded7dc2237382d8074027fe1db6823268d0  2018-08-21 13:40:00 UTC   \n",
       "1       e7fa8035e7ddf75bc0bf5f4a597278afa95d29f3  2018-08-13 11:40:00 UTC   \n",
       "2       3471cc502c2f35682aac256d68cc86083f4bd28f  2018-08-11 18:35:00 UTC   \n",
       "3       b92895e7b640505e4844b475ebf1d76483786365  2018-08-26 17:00:00 UTC   \n",
       "4       b92895e7b640505e4844b475ebf1d76483786365  2018-08-25 12:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "258252  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-08-05 11:35:00 UTC   \n",
       "258253  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-08-05 13:45:00 UTC   \n",
       "258254  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-08-05 12:30:00 UTC   \n",
       "258255  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-08-18 18:10:00 UTC   \n",
       "258256  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-08-21 11:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               686                      722   \n",
       "1          cool          hold               740                      743   \n",
       "2          cool          hold               750                      742   \n",
       "3          cool          auto               711                      709   \n",
       "4          cool          auto               708                      709   \n",
       "...         ...           ...               ...                      ...   \n",
       "258252     cool          auto               752                      760   \n",
       "258253     cool          auto               757                      760   \n",
       "258254     cool          auto               756                      760   \n",
       "258255     cool          auto               757                      760   \n",
       "258256     cool          auto               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           672            MS      Oxford   \n",
       "1                           743            MS    Gulfport   \n",
       "2                           742            MS     Brandon   \n",
       "3                           709            MS      Purvis   \n",
       "4                           709            MS      Purvis   \n",
       "...                         ...           ...         ...   \n",
       "258252                      760            MS  Mooreville   \n",
       "258253                      760            MS  Mooreville   \n",
       "258254                      760            MS  Mooreville   \n",
       "258255                      760            MS  Mooreville   \n",
       "258256                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        26             False         False            False   \n",
       "1                        20             False         False             True   \n",
       "2                        20             False         False            False   \n",
       "3                         0             False         False             True   \n",
       "4                         0             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "258252                   10             False         False            False   \n",
       "258253                   10             False         False            False   \n",
       "258254                   10             False         False            False   \n",
       "258255                   10             False         False            False   \n",
       "258256                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "258252                       Gas  \n",
       "258253                       Gas  \n",
       "258254                       Gas  \n",
       "258255                       Gas  \n",
       "258256                       Gas  \n",
       "\n",
       "[246362 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/MS/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/MS-day/2019-aug-day-MS.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>95dd14ef17a6a1561dea9b0046b0bc9d11b9b9ff</td>\n",
       "      <td>2019-08-22 12:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>781</td>\n",
       "      <td>779</td>\n",
       "      <td>779</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cd632477aa8e4b117fe3fefc978149512e38a1be</td>\n",
       "      <td>2019-08-25 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>716</td>\n",
       "      <td>666</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>295d5108c3ad59096d41bf12a67714ae31641a85</td>\n",
       "      <td>2019-08-07 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>759</td>\n",
       "      <td>646</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f48ff8af79c722f86b46269df5785ce40223e5b2</td>\n",
       "      <td>2019-08-09 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>679</td>\n",
       "      <td>679</td>\n",
       "      <td>MS</td>\n",
       "      <td>Southaven</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>95dd14ef17a6a1561dea9b0046b0bc9d11b9b9ff</td>\n",
       "      <td>2019-08-28 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>779</td>\n",
       "      <td>779</td>\n",
       "      <td>MS</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351468</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2019-08-25 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351469</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2019-08-25 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351470</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2019-08-25 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351471</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2019-08-25 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351472</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2019-08-25 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>339904 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       95dd14ef17a6a1561dea9b0046b0bc9d11b9b9ff  2019-08-22 12:35:00 UTC   \n",
       "1       cd632477aa8e4b117fe3fefc978149512e38a1be  2019-08-25 18:05:00 UTC   \n",
       "2       295d5108c3ad59096d41bf12a67714ae31641a85  2019-08-07 14:35:00 UTC   \n",
       "3       f48ff8af79c722f86b46269df5785ce40223e5b2  2019-08-09 17:20:00 UTC   \n",
       "5       95dd14ef17a6a1561dea9b0046b0bc9d11b9b9ff  2019-08-28 17:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "351468  f0674498c898912150ff0fd0cff9c42ea053a96c  2019-08-25 19:55:00 UTC   \n",
       "351469  f0674498c898912150ff0fd0cff9c42ea053a96c  2019-08-25 18:30:00 UTC   \n",
       "351470  f0674498c898912150ff0fd0cff9c42ea053a96c  2019-08-25 16:45:00 UTC   \n",
       "351471  f0674498c898912150ff0fd0cff9c42ea053a96c  2019-08-25 12:50:00 UTC   \n",
       "351472  f0674498c898912150ff0fd0cff9c42ea053a96c  2019-08-25 16:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               781                      779   \n",
       "1          auto          hold               721                      716   \n",
       "2          cool          auto               763                      759   \n",
       "3          cool          hold               699                      679   \n",
       "5          cool          hold               776                      779   \n",
       "...         ...           ...               ...                      ...   \n",
       "351468     cool          hold               764                      760   \n",
       "351469     cool          hold               757                      760   \n",
       "351470     cool          hold               762                      760   \n",
       "351471     cool          hold               762                      760   \n",
       "351472     cool          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           779            MS    Brandon                    9   \n",
       "1                           666            MS    Clinton                   10   \n",
       "2                           646            MS    Madison                   20   \n",
       "3                           679            MS  Southaven                   10   \n",
       "5                           779            MS    Brandon                    9   \n",
       "...                         ...           ...        ...                  ...   \n",
       "351468                      760            MS    Madison                   10   \n",
       "351469                      760            MS    Madison                   10   \n",
       "351470                      760            MS    Madison                   10   \n",
       "351471                      760            MS    Madison                   10   \n",
       "351472                      760            MS    Madison                   10   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                   True         False            False   \n",
       "1                  False         False            False   \n",
       "2                   True         False            False   \n",
       "3                  False         False            False   \n",
       "5                   True         False            False   \n",
       "...                  ...           ...              ...   \n",
       "351468             False         False            False   \n",
       "351469             False         False            False   \n",
       "351470             False         False            False   \n",
       "351471             False         False            False   \n",
       "351472             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "351468                       Gas  \n",
       "351469                       Gas  \n",
       "351470                       Gas  \n",
       "351471                       Gas  \n",
       "351472                       Gas  \n",
       "\n",
       "[339904 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/MS/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/MS-day/2020-aug-day-MS.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>675453cae9b4ab73fefa9dfcc356a214facb645f</td>\n",
       "      <td>2020-08-08 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>693</td>\n",
       "      <td>693</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12b757b138fde9ee0e12c487691e154ecc131712</td>\n",
       "      <td>2020-08-16 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>745</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f352ad15d20ecc6e685bfa9284c295b5d942d43f</td>\n",
       "      <td>2020-08-31 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>653</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9ecfff8a196424ef9e7409a9f04ddcb676c4866a</td>\n",
       "      <td>2020-08-29 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>714</td>\n",
       "      <td>696</td>\n",
       "      <td>MS</td>\n",
       "      <td>Gulfport</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1c771224a928c91a10a647dc8296505bed7b71fd</td>\n",
       "      <td>2020-08-02 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>803</td>\n",
       "      <td>673</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336776</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2020-08-03 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336777</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2020-08-20 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336778</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2020-08-03 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336779</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2020-08-20 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336780</th>\n",
       "      <td>dc1e40c795a411c773c78d408555dc894f11c103</td>\n",
       "      <td>2020-08-20 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MS</td>\n",
       "      <td>New Albany</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>315074 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       675453cae9b4ab73fefa9dfcc356a214facb645f  2020-08-08 12:55:00 UTC   \n",
       "2       12b757b138fde9ee0e12c487691e154ecc131712  2020-08-16 19:05:00 UTC   \n",
       "3       f352ad15d20ecc6e685bfa9284c295b5d942d43f  2020-08-31 14:10:00 UTC   \n",
       "4       9ecfff8a196424ef9e7409a9f04ddcb676c4866a  2020-08-29 17:35:00 UTC   \n",
       "5       1c771224a928c91a10a647dc8296505bed7b71fd  2020-08-02 19:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "336776  dc1e40c795a411c773c78d408555dc894f11c103  2020-08-03 19:05:00 UTC   \n",
       "336777  dc1e40c795a411c773c78d408555dc894f11c103  2020-08-20 11:40:00 UTC   \n",
       "336778  dc1e40c795a411c773c78d408555dc894f11c103  2020-08-03 12:45:00 UTC   \n",
       "336779  dc1e40c795a411c773c78d408555dc894f11c103  2020-08-20 19:50:00 UTC   \n",
       "336780  dc1e40c795a411c773c78d408555dc894f11c103  2020-08-20 18:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          cool          hold               700                      693   \n",
       "2          cool          auto               754                      750   \n",
       "3          cool          hold               700                      700   \n",
       "4          cool          hold               729                      714   \n",
       "5          auto          hold               775                      803   \n",
       "...         ...           ...               ...                      ...   \n",
       "336776     cool          auto               704                      700   \n",
       "336777     cool          auto               699                      700   \n",
       "336778     cool          auto               704                      700   \n",
       "336779     cool          auto               704                      700   \n",
       "336780     cool          auto               703                      700   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "1                           693            MS     Madison   \n",
       "2                           745            MS     Jackson   \n",
       "3                           653            MS     Madison   \n",
       "4                           696            MS    Gulfport   \n",
       "5                           673            MS     Clinton   \n",
       "...                         ...           ...         ...   \n",
       "336776                      700            MS  New Albany   \n",
       "336777                      700            MS  New Albany   \n",
       "336778                      700            MS  New Albany   \n",
       "336779                      700            MS  New Albany   \n",
       "336780                      700            MS  New Albany   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        10             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        45             False         False            False   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "336776                   10             False         False            False   \n",
       "336777                   10             False         False            False   \n",
       "336778                   10             False         False            False   \n",
       "336779                   10             False         False            False   \n",
       "336780                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "336776                       Gas  \n",
       "336777                       Gas  \n",
       "336778                       Gas  \n",
       "336779                       Gas  \n",
       "336780                       Gas  \n",
       "\n",
       "[315074 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/MS/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>760.175676</td>\n",
       "      <td>767.140541</td>\n",
       "      <td>746.391892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>744.396632</td>\n",
       "      <td>740.381665</td>\n",
       "      <td>740.364827</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>022c67a9cc7538740068f7248330a614c0ddb96f</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>706.000000</td>\n",
       "      <td>739.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>702.380866</td>\n",
       "      <td>703.107100</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04e9b5167752b184a42353cfbf667d3e89d98f33</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Long Beach</td>\n",
       "      <td>741.443709</td>\n",
       "      <td>746.624724</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>765.852201</td>\n",
       "      <td>774.937107</td>\n",
       "      <td>774.937107</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>402</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Madison</td>\n",
       "      <td>691.906475</td>\n",
       "      <td>690.446043</td>\n",
       "      <td>689.841727</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>726.074627</td>\n",
       "      <td>729.000000</td>\n",
       "      <td>725.604478</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>735.812931</td>\n",
       "      <td>733.133015</td>\n",
       "      <td>733.133015</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>731.527462</td>\n",
       "      <td>726.302083</td>\n",
       "      <td>726.302083</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1348 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   aug  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   aug  2017     cool   \n",
       "2    022c67a9cc7538740068f7248330a614c0ddb96f   aug  2017     auto   \n",
       "3    0442aded7dc2237382d8074027fe1db6823268d0   aug  2017     auto   \n",
       "4    04e9b5167752b184a42353cfbf667d3e89d98f33   aug  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "401  fe9bc276b524605ee152e4051285e4e080e77a3f   aug  2020     cool   \n",
       "402  ff5551a641dc7446df1914428ded425dde839080   aug  2020     cool   \n",
       "403  ff5551a641dc7446df1914428ded425dde839080   aug  2020     cool   \n",
       "404  ffe938566049e7ad2eb8c1726ec2215bfda979d7   aug  2020     cool   \n",
       "405  ffe938566049e7ad2eb8c1726ec2215bfda979d7   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            760.175676   \n",
       "1            hold  Ocean Springs            744.396632   \n",
       "2            auto         Biloxi            706.000000   \n",
       "3            hold         Oxford            702.380866   \n",
       "4            auto     Long Beach            741.443709   \n",
       "..            ...            ...                   ...   \n",
       "401          hold         Tupelo            765.852201   \n",
       "402          auto        Madison            691.906475   \n",
       "403          hold        Madison            726.074627   \n",
       "404          auto     Mooreville            735.812931   \n",
       "405          hold     Mooreville            731.527462   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     767.140541                   746.391892   \n",
       "1                     740.381665                   740.364827   \n",
       "2                     739.000000                   670.000000   \n",
       "3                     703.107100                   640.000000   \n",
       "4                     746.624724                   680.000000   \n",
       "..                           ...                          ...   \n",
       "401                   774.937107                   774.937107   \n",
       "402                   690.446043                   689.841727   \n",
       "403                   729.000000                   725.604478   \n",
       "404                   733.133015                   733.133015   \n",
       "405                   726.302083                   726.302083   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                    5.0              True         False             True  \n",
       "3                   26.0             False         False            False  \n",
       "4                    5.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "401                 75.0             False         False            False  \n",
       "402                 30.0             False         False            False  \n",
       "403                 30.0             False         False            False  \n",
       "404                 10.0             False         False            False  \n",
       "405                 10.0             False         False            False  \n",
       "\n",
       "[1348 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/aug/\" + file)\n",
    "    MS_aug = pd.concat([MS_aug, df])\n",
    "    \n",
    "MS_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_aug.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/MS-day/2017-dec-day-MS.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>024d1624a424d57f4cdc19a5fa9aa6d90739ef33</td>\n",
       "      <td>2017-12-22 11:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Horn Lake</td>\n",
       "      <td>26</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6958304af4064a7f1e351b21c0ee3d85f30d02cb</td>\n",
       "      <td>2017-12-23 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>728</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9b718599ed03efd98e2b15948c86641a7cd4b249</td>\n",
       "      <td>2017-12-23 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>795</td>\n",
       "      <td>745</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0cc861d81926d70c036828385eb0e61b6e98dfaa</td>\n",
       "      <td>2017-12-06 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>797</td>\n",
       "      <td>702</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6958304af4064a7f1e351b21c0ee3d85f30d02cb</td>\n",
       "      <td>2017-12-03 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>773</td>\n",
       "      <td>723</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167508</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-12-23 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>776</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167509</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-12-23 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>771</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167510</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-12-07 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167511</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-12-23 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167512</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2017-12-07 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>161711 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       024d1624a424d57f4cdc19a5fa9aa6d90739ef33  2017-12-22 11:50:00 UTC   \n",
       "1       6958304af4064a7f1e351b21c0ee3d85f30d02cb  2017-12-23 14:35:00 UTC   \n",
       "2       9b718599ed03efd98e2b15948c86641a7cd4b249  2017-12-23 19:10:00 UTC   \n",
       "4       0cc861d81926d70c036828385eb0e61b6e98dfaa  2017-12-06 12:50:00 UTC   \n",
       "5       6958304af4064a7f1e351b21c0ee3d85f30d02cb  2017-12-03 16:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "167508  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-12-23 15:45:00 UTC   \n",
       "167509  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-12-23 15:25:00 UTC   \n",
       "167510  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-12-07 16:50:00 UTC   \n",
       "167511  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-12-23 15:40:00 UTC   \n",
       "167512  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2017-12-07 18:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               729                      785   \n",
       "1          auto          auto               728                      785   \n",
       "2          heat          hold               740                      795   \n",
       "4          heat          auto               731                      797   \n",
       "5          auto          hold               722                      773   \n",
       "...         ...           ...               ...                      ...   \n",
       "167508     heat          auto               776                      760   \n",
       "167509     heat          auto               771                      760   \n",
       "167510     heat          auto               763                      760   \n",
       "167511     heat          auto               775                      760   \n",
       "167512     heat          auto               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           735            MS   Horn Lake   \n",
       "1                           735            MS      Oxford   \n",
       "2                           745            MS     Madison   \n",
       "4                           702            MS     Madison   \n",
       "5                           723            MS      Oxford   \n",
       "...                         ...           ...         ...   \n",
       "167508                      760            MS  Mooreville   \n",
       "167509                      760            MS  Mooreville   \n",
       "167510                      760            MS  Mooreville   \n",
       "167511                      760            MS  Mooreville   \n",
       "167512                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        26              True         False             True   \n",
       "1                         5             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "5                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "167508                   10             False         False            False   \n",
       "167509                   10             False         False            False   \n",
       "167510                   10             False         False            False   \n",
       "167511                   10             False         False            False   \n",
       "167512                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "167508                       Gas  \n",
       "167509                       Gas  \n",
       "167510                       Gas  \n",
       "167511                       Gas  \n",
       "167512                       Gas  \n",
       "\n",
       "[161711 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/MS/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/MS-day/2018-dec-day-MS.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1b6fe47c57f6e62dba5ace2d6a2504776c43f87f</td>\n",
       "      <td>2018-12-15 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>745</td>\n",
       "      <td>677</td>\n",
       "      <td>MS</td>\n",
       "      <td>Olive Branch</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2018-12-23 13:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>722</td>\n",
       "      <td>642</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6958304af4064a7f1e351b21c0ee3d85f30d02cb</td>\n",
       "      <td>2018-12-07 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>759</td>\n",
       "      <td>659</td>\n",
       "      <td>MS</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d23049697dc10f7bda2995dcd45a259452e331c5</td>\n",
       "      <td>2018-12-09 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>742</td>\n",
       "      <td>682</td>\n",
       "      <td>MS</td>\n",
       "      <td>Belden</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db2579b46576227a29f30401612ae2a9f036935c</td>\n",
       "      <td>2018-12-16 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>762</td>\n",
       "      <td>692</td>\n",
       "      <td>MS</td>\n",
       "      <td>Vicksburg</td>\n",
       "      <td>65</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295044</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-12-05 10:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295045</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-12-05 07:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295046</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-12-09 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295047</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-12-09 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295048</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2018-12-09 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>283277 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1b6fe47c57f6e62dba5ace2d6a2504776c43f87f  2018-12-15 18:55:00 UTC   \n",
       "1       d23049697dc10f7bda2995dcd45a259452e331c5  2018-12-23 13:05:00 UTC   \n",
       "2       6958304af4064a7f1e351b21c0ee3d85f30d02cb  2018-12-07 16:45:00 UTC   \n",
       "3       d23049697dc10f7bda2995dcd45a259452e331c5  2018-12-09 19:55:00 UTC   \n",
       "4       db2579b46576227a29f30401612ae2a9f036935c  2018-12-16 14:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "295044  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-12-05 10:45:00 UTC   \n",
       "295045  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-12-05 07:45:00 UTC   \n",
       "295046  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-12-09 16:25:00 UTC   \n",
       "295047  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-12-09 16:05:00 UTC   \n",
       "295048  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2018-12-09 15:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               703                      745   \n",
       "1          auto          hold               654                      722   \n",
       "2          auto          hold               715                      759   \n",
       "3          auto          hold               684                      742   \n",
       "4          auto          hold               705                      762   \n",
       "...         ...           ...               ...                      ...   \n",
       "295044     heat          auto               762                      760   \n",
       "295045     heat          auto               763                      760   \n",
       "295046     heat          auto               758                      760   \n",
       "295047     heat          auto               755                      760   \n",
       "295048     heat          auto               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           677            MS  Olive Branch   \n",
       "1                           642            MS        Belden   \n",
       "2                           659            MS        Oxford   \n",
       "3                           682            MS        Belden   \n",
       "4                           692            MS     Vicksburg   \n",
       "...                         ...           ...           ...   \n",
       "295044                      760            MS    Mooreville   \n",
       "295045                      760            MS    Mooreville   \n",
       "295046                      760            MS    Mooreville   \n",
       "295047                      760            MS    Mooreville   \n",
       "295048                      760            MS    Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                        65              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "295044                   10             False         False            False   \n",
       "295045                   10             False         False            False   \n",
       "295046                   10             False         False            False   \n",
       "295047                   10             False         False            False   \n",
       "295048                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "295044                       Gas  \n",
       "295045                       Gas  \n",
       "295046                       Gas  \n",
       "295047                       Gas  \n",
       "295048                       Gas  \n",
       "\n",
       "[283277 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/MS/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/MS-day/2019-dec-day-MS.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-12-30 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16daf0ca4bfcc11cc2e4721db2c14c38559c0a92</td>\n",
       "      <td>2019-12-30 13:45:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>MS</td>\n",
       "      <td>Amory</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f47b3881ab449632a3dfc5d0b96cc84335174dd3</td>\n",
       "      <td>2019-12-20 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>750</td>\n",
       "      <td>693</td>\n",
       "      <td>MS</td>\n",
       "      <td>Petal</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f918d405d012b701c0cb10dd807539f0a731ceea</td>\n",
       "      <td>2019-12-23 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>743</td>\n",
       "      <td>717</td>\n",
       "      <td>MS</td>\n",
       "      <td>Pass Christian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>71fc691a191d2bbea3ea5b823d23477631bb8ea4</td>\n",
       "      <td>2019-12-12 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>669</td>\n",
       "      <td>669</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ecru</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307340</th>\n",
       "      <td>748c069371486cb31498d4fa5bec2a5949d72e83</td>\n",
       "      <td>2019-12-21 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307341</th>\n",
       "      <td>41db72abb39a21e919affae19ff3413510b13633</td>\n",
       "      <td>2019-12-29 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307342</th>\n",
       "      <td>8d5497d8eeb1d76fa99ce5d78b2fa338d679d805</td>\n",
       "      <td>2019-12-29 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307343</th>\n",
       "      <td>8d5497d8eeb1d76fa99ce5d78b2fa338d679d805</td>\n",
       "      <td>2019-12-29 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307344</th>\n",
       "      <td>8d5497d8eeb1d76fa99ce5d78b2fa338d679d805</td>\n",
       "      <td>2019-12-29 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>297829 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "2       f918d405d012b701c0cb10dd807539f0a731ceea  2019-12-30 13:30:00 UTC   \n",
       "3       16daf0ca4bfcc11cc2e4721db2c14c38559c0a92  2019-12-30 13:45:00 UTC   \n",
       "4       f47b3881ab449632a3dfc5d0b96cc84335174dd3  2019-12-20 15:55:00 UTC   \n",
       "5       f918d405d012b701c0cb10dd807539f0a731ceea  2019-12-23 14:50:00 UTC   \n",
       "8       71fc691a191d2bbea3ea5b823d23477631bb8ea4  2019-12-12 17:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "307340  748c069371486cb31498d4fa5bec2a5949d72e83  2019-12-21 15:30:00 UTC   \n",
       "307341  41db72abb39a21e919affae19ff3413510b13633  2019-12-29 16:15:00 UTC   \n",
       "307342  8d5497d8eeb1d76fa99ce5d78b2fa338d679d805  2019-12-29 19:00:00 UTC   \n",
       "307343  8d5497d8eeb1d76fa99ce5d78b2fa338d679d805  2019-12-29 18:55:00 UTC   \n",
       "307344  8d5497d8eeb1d76fa99ce5d78b2fa338d679d805  2019-12-29 19:15:00 UTC   \n",
       "\n",
       "           HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "2              heat          auto               723                      743   \n",
       "3       auxHeatOnly          hold               687                      677   \n",
       "4              auto          hold               689                      750   \n",
       "5              heat          auto               718                      743   \n",
       "8              heat          hold               661                      669   \n",
       "...             ...           ...               ...                      ...   \n",
       "307340         auto          hold               696                      800   \n",
       "307341         cool          auto               719                      750   \n",
       "307342         cool          auto               761                      760   \n",
       "307343         cool          auto               761                      760   \n",
       "307344         cool          auto               762                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "2                           717            MS  Pass Christian   \n",
       "3                           677            MS           Amory   \n",
       "4                           693            MS           Petal   \n",
       "5                           717            MS  Pass Christian   \n",
       "8                           669            MS            Ecru   \n",
       "...                         ...           ...             ...   \n",
       "307340                      750            MS   Ocean Springs   \n",
       "307341                      750            MS   Ocean Springs   \n",
       "307342                      760            MS   Ocean Springs   \n",
       "307343                      760            MS   Ocean Springs   \n",
       "307344                      760            MS   Ocean Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "2                        10             False         False             True   \n",
       "3                        10              True         False             True   \n",
       "4                        10             False         False            False   \n",
       "5                        10             False         False             True   \n",
       "8                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "307340                   19             False         False            False   \n",
       "307341                   15             False         False            False   \n",
       "307342                   10             False         False             True   \n",
       "307343                   10             False         False             True   \n",
       "307344                   10             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "5                       Electric  \n",
       "8                            Gas  \n",
       "...                          ...  \n",
       "307340                       Gas  \n",
       "307341                       Gas  \n",
       "307342                  Electric  \n",
       "307343                  Electric  \n",
       "307344                  Electric  \n",
       "\n",
       "[297829 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/MS/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/MS-day/2020-dec-day-MS.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>629f03d3cff1523e3f45e61248076717d308fe72</td>\n",
       "      <td>2020-12-27 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>MS</td>\n",
       "      <td>Clinton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aa17f4db1d39aee255374137e60637b4b46314db</td>\n",
       "      <td>2020-12-13 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>720</td>\n",
       "      <td>662</td>\n",
       "      <td>MS</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>77045d611c3cf60d028145a458d955bd64f29192</td>\n",
       "      <td>2020-12-15 13:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>MS</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b97e267eaef40bd27c8fff6a536a0deb3ea69821</td>\n",
       "      <td>2020-12-04 11:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>656</td>\n",
       "      <td>661</td>\n",
       "      <td>661</td>\n",
       "      <td>MS</td>\n",
       "      <td>Purvis</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4</td>\n",
       "      <td>2020-12-08 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>754</td>\n",
       "      <td>704</td>\n",
       "      <td>MS</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270866</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2020-12-13 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270867</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2020-12-13 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270868</th>\n",
       "      <td>f0674498c898912150ff0fd0cff9c42ea053a96c</td>\n",
       "      <td>2020-12-13 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Madison</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270869</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2020-12-19 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270870</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>2020-12-19 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MS</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>262786 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       629f03d3cff1523e3f45e61248076717d308fe72  2020-12-27 17:15:00 UTC   \n",
       "1       aa17f4db1d39aee255374137e60637b4b46314db  2020-12-13 16:30:00 UTC   \n",
       "2       77045d611c3cf60d028145a458d955bd64f29192  2020-12-15 13:00:00 UTC   \n",
       "3       b97e267eaef40bd27c8fff6a536a0deb3ea69821  2020-12-04 11:10:00 UTC   \n",
       "5       1acb7c5c1a18c3a6baf8917710d9b20eab9e2ac4  2020-12-08 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "270866  f0674498c898912150ff0fd0cff9c42ea053a96c  2020-12-13 13:45:00 UTC   \n",
       "270867  f0674498c898912150ff0fd0cff9c42ea053a96c  2020-12-13 19:20:00 UTC   \n",
       "270868  f0674498c898912150ff0fd0cff9c42ea053a96c  2020-12-13 18:05:00 UTC   \n",
       "270869  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2020-12-19 15:45:00 UTC   \n",
       "270870  ffe938566049e7ad2eb8c1726ec2215bfda979d7  2020-12-19 16:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               689                      692   \n",
       "1          auto          hold               661                      720   \n",
       "2          heat          hold               689                      682   \n",
       "3          heat          hold               656                      661   \n",
       "5          auto          hold               715                      754   \n",
       "...         ...           ...               ...                      ...   \n",
       "270866     heat          auto               756                      760   \n",
       "270867     heat          auto               759                      760   \n",
       "270868     heat          auto               752                      760   \n",
       "270869     heat          hold               757                      760   \n",
       "270870     heat          hold               757                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           692            MS     Clinton   \n",
       "1                           662            MS     Jackson   \n",
       "2                           682            MS      Biloxi   \n",
       "3                           661            MS      Purvis   \n",
       "5                           704            MS      Tupelo   \n",
       "...                         ...           ...         ...   \n",
       "270866                      760            MS     Madison   \n",
       "270867                      760            MS     Madison   \n",
       "270868                      760            MS     Madison   \n",
       "270869                      760            MS  Mooreville   \n",
       "270870                      760            MS  Mooreville   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                        15             False         False            False   \n",
       "3                        10              True         False             True   \n",
       "5                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "270866                   10             False         False            False   \n",
       "270867                   10             False         False            False   \n",
       "270868                   10             False         False            False   \n",
       "270869                   10             False         False            False   \n",
       "270870                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "270866                       Gas  \n",
       "270867                       Gas  \n",
       "270868                       Gas  \n",
       "270869                       Gas  \n",
       "270870                       Gas  \n",
       "\n",
       "[262786 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/MS/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MS/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>732.222222</td>\n",
       "      <td>735.559524</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>723.419192</td>\n",
       "      <td>735.151515</td>\n",
       "      <td>735.151515</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>710.068783</td>\n",
       "      <td>714.227513</td>\n",
       "      <td>710.695767</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>697.422996</td>\n",
       "      <td>695.005274</td>\n",
       "      <td>695.005274</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>022c67a9cc7538740068f7248330a614c0ddb96f</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>731.164557</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Madison</td>\n",
       "      <td>674.104167</td>\n",
       "      <td>680.562500</td>\n",
       "      <td>679.791667</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>659.727273</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>735.111111</td>\n",
       "      <td>703.000000</td>\n",
       "      <td>703.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>732.797885</td>\n",
       "      <td>734.270270</td>\n",
       "      <td>734.270270</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>735.199386</td>\n",
       "      <td>736.962375</td>\n",
       "      <td>736.962375</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1953 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01194744830da7bbc9603d51735c58faef66f72e   dec  2017     cool   \n",
       "1    01194744830da7bbc9603d51735c58faef66f72e   dec  2017     cool   \n",
       "2    01194744830da7bbc9603d51735c58faef66f72e   dec  2017     heat   \n",
       "3    01194744830da7bbc9603d51735c58faef66f72e   dec  2017     heat   \n",
       "4    022c67a9cc7538740068f7248330a614c0ddb96f   dec  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "481  ff5551a641dc7446df1914428ded425dde839080   dec  2020     heat   \n",
       "482  ff5551a641dc7446df1914428ded425dde839080   dec  2020     heat   \n",
       "483  ffe938566049e7ad2eb8c1726ec2215bfda979d7   dec  2020     cool   \n",
       "484  ffe938566049e7ad2eb8c1726ec2215bfda979d7   dec  2020     heat   \n",
       "485  ffe938566049e7ad2eb8c1726ec2215bfda979d7   dec  2020     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto  Ocean Springs            732.222222   \n",
       "1            hold  Ocean Springs            723.419192   \n",
       "2            auto  Ocean Springs            710.068783   \n",
       "3            hold  Ocean Springs            697.422996   \n",
       "4            auto         Biloxi            731.164557   \n",
       "..            ...            ...                   ...   \n",
       "481          auto        Madison            674.104167   \n",
       "482          hold        Madison            659.727273   \n",
       "483          hold     Mooreville            735.111111   \n",
       "484          auto     Mooreville            732.797885   \n",
       "485          hold     Mooreville            735.199386   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     735.559524                   710.000000   \n",
       "1                     735.151515                   735.151515   \n",
       "2                     714.227513                   710.695767   \n",
       "3                     695.005274                   695.005274   \n",
       "4                     780.000000                   730.000000   \n",
       "..                           ...                          ...   \n",
       "481                   680.562500                   679.791667   \n",
       "482                   680.000000                   680.000000   \n",
       "483                   703.000000                   703.000000   \n",
       "484                   734.270270                   734.270270   \n",
       "485                   736.962375                   736.962375   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    0.0             False         False             True  \n",
       "1                    0.0             False         False             True  \n",
       "2                    0.0             False         False             True  \n",
       "3                    0.0             False         False             True  \n",
       "4                    5.0              True         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "481                 30.0             False         False            False  \n",
       "482                 30.0             False         False            False  \n",
       "483                 10.0             False         False            False  \n",
       "484                 10.0             False         False            False  \n",
       "485                 10.0             False         False            False  \n",
       "\n",
       "[1953 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MS/dec/\" + file)\n",
    "    MS_dec = pd.concat([MS_dec, df])\n",
    "    \n",
    "MS_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_dec.to_csv(\"Scraper_Output/State_Month_Day/MS/MS_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/MS/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>760.175676</td>\n",
       "      <td>767.140541</td>\n",
       "      <td>746.391892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01194744830da7bbc9603d51735c58faef66f72e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ocean Springs</td>\n",
       "      <td>744.396632</td>\n",
       "      <td>740.381665</td>\n",
       "      <td>740.364827</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>022c67a9cc7538740068f7248330a614c0ddb96f</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Biloxi</td>\n",
       "      <td>706.000000</td>\n",
       "      <td>739.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0442aded7dc2237382d8074027fe1db6823268d0</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oxford</td>\n",
       "      <td>702.380866</td>\n",
       "      <td>703.107100</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04e9b5167752b184a42353cfbf667d3e89d98f33</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Long Beach</td>\n",
       "      <td>741.443709</td>\n",
       "      <td>746.624724</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1603</th>\n",
       "      <td>fdfe0112380a8ee36c3dfe16899298a72c886d49</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>731.842051</td>\n",
       "      <td>730.814017</td>\n",
       "      <td>730.814017</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1604</th>\n",
       "      <td>fe9bc276b524605ee152e4051285e4e080e77a3f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tupelo</td>\n",
       "      <td>709.118644</td>\n",
       "      <td>705.403955</td>\n",
       "      <td>705.403955</td>\n",
       "      <td>75.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1605</th>\n",
       "      <td>ff5551a641dc7446df1914428ded425dde839080</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Madison</td>\n",
       "      <td>708.410112</td>\n",
       "      <td>716.943820</td>\n",
       "      <td>715.078652</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1606</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>733.231047</td>\n",
       "      <td>731.041742</td>\n",
       "      <td>731.041742</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1607</th>\n",
       "      <td>ffe938566049e7ad2eb8c1726ec2215bfda979d7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mooreville</td>\n",
       "      <td>741.214286</td>\n",
       "      <td>731.285714</td>\n",
       "      <td>731.285714</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10258 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     01194744830da7bbc9603d51735c58faef66f72e   aug  2017     cool   \n",
       "1     01194744830da7bbc9603d51735c58faef66f72e   aug  2017     cool   \n",
       "2     022c67a9cc7538740068f7248330a614c0ddb96f   aug  2017     auto   \n",
       "3     0442aded7dc2237382d8074027fe1db6823268d0   aug  2017     auto   \n",
       "4     04e9b5167752b184a42353cfbf667d3e89d98f33   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1603  fdfe0112380a8ee36c3dfe16899298a72c886d49   jun  2021     cool   \n",
       "1604  fe9bc276b524605ee152e4051285e4e080e77a3f   jun  2021     cool   \n",
       "1605  ff5551a641dc7446df1914428ded425dde839080   jun  2021     cool   \n",
       "1606  ffe938566049e7ad2eb8c1726ec2215bfda979d7   jun  2021     cool   \n",
       "1607  ffe938566049e7ad2eb8c1726ec2215bfda979d7   jun  2021     heat   \n",
       "\n",
       "     CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0             auto  Ocean Springs            760.175676   \n",
       "1             hold  Ocean Springs            744.396632   \n",
       "2             auto         Biloxi            706.000000   \n",
       "3             hold         Oxford            702.380866   \n",
       "4             auto     Long Beach            741.443709   \n",
       "...            ...            ...                   ...   \n",
       "1603          hold        Jackson            731.842051   \n",
       "1604          hold         Tupelo            709.118644   \n",
       "1605          hold        Madison            708.410112   \n",
       "1606          hold     Mooreville            733.231047   \n",
       "1607          hold     Mooreville            741.214286   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      767.140541                   746.391892   \n",
       "1                      740.381665                   740.364827   \n",
       "2                      739.000000                   670.000000   \n",
       "3                      703.107100                   640.000000   \n",
       "4                      746.624724                   680.000000   \n",
       "...                           ...                          ...   \n",
       "1603                   730.814017                   730.814017   \n",
       "1604                   705.403955                   705.403955   \n",
       "1605                   716.943820                   715.078652   \n",
       "1606                   731.041742                   731.041742   \n",
       "1607                   731.285714                   731.285714   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False             True  \n",
       "1                     0.0             False         False             True  \n",
       "2                     5.0              True         False             True  \n",
       "3                    26.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1603                 30.0             False         False            False  \n",
       "1604                 75.0             False         False            False  \n",
       "1605                 30.0             False         False            False  \n",
       "1606                 10.0             False         False            False  \n",
       "1607                 10.0             False         False            False  \n",
       "\n",
       "[10258 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MS_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/MS/\" + file)\n",
    "    MS_all = pd.concat([MS_all, df])\n",
    "    \n",
    "MS_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "MS_all.to_csv(\"Scraper_Output/State_Month_Day/MS_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['MS']\n",
      "Unique jan_2018: ['MS']\n",
      "Unique jan_2019: ['MS']\n",
      "Unique jan_2020: ['MS']\n",
      "Unique jan_2021: ['MS']\n",
      "Unique feb_2017: ['MS']\n",
      "Unique feb_2018: ['MS']\n",
      "Unique feb_2019: ['MS']\n",
      "Unique feb_2020: ['MS']\n",
      "Unique feb_2021: ['MS']\n",
      "Unique jun_2017: ['MS']\n",
      "Unique jun_2018: ['MS']\n",
      "Unique jun_2019: ['MS']\n",
      "Unique jun_2020: ['MS']\n",
      "Unique jun_2021: ['MS']\n",
      "Unique jul_2017: ['MS']\n",
      "Unique jul_2018: ['MS']\n",
      "Unique jul_2019: ['MS']\n",
      "Unique jul_2020: ['MS']\n",
      "Unique jul_2021: ['MS']\n",
      "Unique aug_2017: ['MS']\n",
      "Unique aug_2018: ['MS']\n",
      "Unique aug_2019: ['MS']\n",
      "Unique aug_2020: ['MS']\n",
      "Unique dec_2017: ['MS']\n",
      "Unique dec_2018: ['MS']\n",
      "Unique dec_2019: ['MS']\n",
      "Unique dec_2020: ['MS']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mMSe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}