DYD_CT-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/CT-day/2017-jan-day-CT.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9b51dbf3db985732926fa306564f61277b9b6bfa</td>\n",
       "      <td>2017-01-28 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>704</td>\n",
       "      <td>704</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2017-01-24 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>639</td>\n",
       "      <td>650</td>\n",
       "      <td>627</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>90311ac9900b2ba8a03e5f6348c84a144db460b1</td>\n",
       "      <td>2017-01-31 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>623</td>\n",
       "      <td>650</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Mansfield</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>79c1501945e02406136a1cfaba35a338a2727fc5</td>\n",
       "      <td>2017-01-07 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>668</td>\n",
       "      <td>792</td>\n",
       "      <td>606</td>\n",
       "      <td>CT</td>\n",
       "      <td>killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ecc6ddba35a42fcb31e3c9649a554a96037d34b5</td>\n",
       "      <td>2017-01-27 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>800</td>\n",
       "      <td>605</td>\n",
       "      <td>CT</td>\n",
       "      <td>Redding</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118741</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-01-19 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118742</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-01-27 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118743</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2017-01-19 11:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118744</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2017-01-19 12:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118745</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2017-01-19 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111418 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       9b51dbf3db985732926fa306564f61277b9b6bfa  2017-01-28 19:50:00 UTC   \n",
       "5       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2017-01-24 17:25:00 UTC   \n",
       "6       90311ac9900b2ba8a03e5f6348c84a144db460b1  2017-01-31 19:30:00 UTC   \n",
       "7       79c1501945e02406136a1cfaba35a338a2727fc5  2017-01-07 17:25:00 UTC   \n",
       "8       ecc6ddba35a42fcb31e3c9649a554a96037d34b5  2017-01-27 12:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "118741  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-01-19 14:05:00 UTC   \n",
       "118742  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-01-27 12:50:00 UTC   \n",
       "118743  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2017-01-19 11:30:00 UTC   \n",
       "118744  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2017-01-19 12:15:00 UTC   \n",
       "118745  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2017-01-19 12:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               674                      704   \n",
       "5          heat          hold               639                      650   \n",
       "6          heat          hold               623                      650   \n",
       "7          heat          auto               668                      792   \n",
       "8          heat          hold               668                      800   \n",
       "...         ...           ...               ...                      ...   \n",
       "118741     heat          hold               758                      760   \n",
       "118742     heat          hold               757                      760   \n",
       "118743     heat          hold               760                      760   \n",
       "118744     heat          hold               752                      760   \n",
       "118745     heat          hold               761                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           704            CT         Orange   \n",
       "5                           627            CT  Windsor Locks   \n",
       "6                           635            CT      Mansfield   \n",
       "7                           606            CT   killingworth   \n",
       "8                           605            CT        Redding   \n",
       "...                         ...           ...            ...   \n",
       "118741                      760            CT        Norwalk   \n",
       "118742                      760            CT        Norwalk   \n",
       "118743                      760            CT    New Milford   \n",
       "118744                      760            CT    New Milford   \n",
       "118745                      760            CT    New Milford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        45             False         False            False   \n",
       "5                        30             False         False            False   \n",
       "6                        20              True         False             True   \n",
       "7                        15             False         False            False   \n",
       "8                        30              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "118741                    5             False         False            False   \n",
       "118742                    5             False         False            False   \n",
       "118743                   15             False         False            False   \n",
       "118744                   15             False         False            False   \n",
       "118745                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "5                            Gas  \n",
       "6                       Electric  \n",
       "7                            Gas  \n",
       "8                            Gas  \n",
       "...                          ...  \n",
       "118741                       Gas  \n",
       "118742                       Gas  \n",
       "118743                       Gas  \n",
       "118744                       Gas  \n",
       "118745                       Gas  \n",
       "\n",
       "[111418 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>01b5776ab24d14ace90a8296c49941a4ba8b0173</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>New MIlford</th>\n",
       "      <td>683.484731</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>685.509677</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>05d474a41ce90b51ae9627f1dc2a5214d67b2741</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Guilford</th>\n",
       "      <td>630.937500</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>620.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0a751de00d076714c9f1c136f2203fae474b7ace</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>East Hampton</th>\n",
       "      <td>652.627729</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b3c9b534cf69257d66a1102858406468dd61627</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Greenwich</th>\n",
       "      <td>665.114754</td>\n",
       "      <td>665.004918</td>\n",
       "      <td>664.940984</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Hamden</th>\n",
       "      <td>693.103448</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.462644</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">f91e491da4d3c5ed8917b116ec11e55dfd5104b8</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>killingworth</th>\n",
       "      <td>654.150000</td>\n",
       "      <td>753.300000</td>\n",
       "      <td>626.350000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>killingworth</th>\n",
       "      <td>658.326087</td>\n",
       "      <td>668.195652</td>\n",
       "      <td>663.173913</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Wilton</th>\n",
       "      <td>671.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fc58d08fd7038c94b9cfcb3287a2bbb6e095745f</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Shelton</th>\n",
       "      <td>684.416667</td>\n",
       "      <td>745.416667</td>\n",
       "      <td>693.416667</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fefe9a1e7cdca41851d594cf74b2d28a98aca1de</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Plantsville</th>\n",
       "      <td>684.591153</td>\n",
       "      <td>685.953083</td>\n",
       "      <td>685.887399</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>226 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                         Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford             683.484731   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                630.937500   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton            652.627729   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich               665.114754   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                  693.103448   \n",
       "...                                                                                                       ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth            654.150000   \n",
       "                                                             hold          killingworth            658.326087   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                  671.000000   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                 684.416667   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville             684.591153   \n",
       "\n",
       "                                                                                         TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford                    800.000000   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                       650.000000   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton                   680.000000   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich                      665.004918   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                         760.000000   \n",
       "...                                                                                                              ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth                   753.300000   \n",
       "                                                             hold          killingworth                   668.195652   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                         660.000000   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                        745.416667   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville                    685.953083   \n",
       "\n",
       "                                                                                         TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford                    685.509677   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                       620.000000   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton                   640.000000   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich                      664.940984   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                         680.462644   \n",
       "...                                                                                                              ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth                   626.350000   \n",
       "                                                             hold          killingworth                   663.173913   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                         660.000000   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                        693.416667   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville                    685.887399   \n",
       "\n",
       "                                                                                         Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford                   5.0   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                     35.0   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton                  0.0   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich                     0.0   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                       25.0   \n",
       "...                                                                                                      ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth                  0.0   \n",
       "                                                             hold          killingworth                  0.0   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                        0.0   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                      25.0   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville                  10.0   \n",
       "\n",
       "                                                                                         allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                             \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford              False   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                 False   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton             False   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich                False   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                   False   \n",
       "...                                                                                                   ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth             False   \n",
       "                                                             hold          killingworth             False   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                   False   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                  False   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville              False   \n",
       "\n",
       "                                                                                         Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                         \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford          False   \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford             False   \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton         False   \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich            False   \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden               False   \n",
       "...                                                                                               ...   \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth         False   \n",
       "                                                             hold          killingworth         False   \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton               False   \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton              False   \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville          False   \n",
       "\n",
       "                                                                                         Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                           \n",
       "01b5776ab24d14ace90a8296c49941a4ba8b0173 Jan   2017 auto     hold          New MIlford             False  \n",
       "05d474a41ce90b51ae9627f1dc2a5214d67b2741 Jan   2017 heat     auto          Guilford                False  \n",
       "0a751de00d076714c9f1c136f2203fae474b7ace Jan   2017 heat     auto          East Hampton            False  \n",
       "0b3c9b534cf69257d66a1102858406468dd61627 Jan   2017 heat     auto          Greenwich               False  \n",
       "0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf Jan   2017 heat     auto          Hamden                  False  \n",
       "...                                                                                                  ...  \n",
       "f91e491da4d3c5ed8917b116ec11e55dfd5104b8 Jan   2017 heat     auto          killingworth            False  \n",
       "                                                             hold          killingworth            False  \n",
       "fb4ba55769c3bfb68a844bfd68c7b38ca7e1d559 Jan   2017 heat     hold          Wilton                  False  \n",
       "fc58d08fd7038c94b9cfcb3287a2bbb6e095745f Jan   2017 auto     auto          Shelton                 False  \n",
       "fefe9a1e7cdca41851d594cf74b2d28a98aca1de Jan   2017 heat     hold          Plantsville             False  \n",
       "\n",
       "[226 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/CT/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/CT-day/2018-jan-day-CT.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2018-01-25 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2018-01-25 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4c5caa1e348acf2719f577d8a4727442ca33b60d</td>\n",
       "      <td>2018-01-02 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>740</td>\n",
       "      <td>682</td>\n",
       "      <td>CT</td>\n",
       "      <td>Barkhamsted</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2018-01-25 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0b8e511ebb362a92c2b02469a1887fd2a1f86998</td>\n",
       "      <td>2018-01-13 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>650</td>\n",
       "      <td>745</td>\n",
       "      <td>613</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371224</th>\n",
       "      <td>81b9f64b6f86312ce534fc4a17536a498b3d48a6</td>\n",
       "      <td>2018-01-06 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>East Hartford</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371225</th>\n",
       "      <td>81b9f64b6f86312ce534fc4a17536a498b3d48a6</td>\n",
       "      <td>2018-01-06 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>East Hartford</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371226</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2018-01-02 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371227</th>\n",
       "      <td>4eb272b24b720ab73a573b5d6c69346a7ccac79f</td>\n",
       "      <td>2018-01-15 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371228</th>\n",
       "      <td>4eb272b24b720ab73a573b5d6c69346a7ccac79f</td>\n",
       "      <td>2018-01-15 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>343575 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       0382f14f425de2d9f9645def17ee3cfeada55f78  2018-01-25 12:20:00 UTC   \n",
       "3       0382f14f425de2d9f9645def17ee3cfeada55f78  2018-01-25 12:50:00 UTC   \n",
       "5       4c5caa1e348acf2719f577d8a4727442ca33b60d  2018-01-02 14:15:00 UTC   \n",
       "7       0382f14f425de2d9f9645def17ee3cfeada55f78  2018-01-25 13:05:00 UTC   \n",
       "8       0b8e511ebb362a92c2b02469a1887fd2a1f86998  2018-01-13 15:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "371224  81b9f64b6f86312ce534fc4a17536a498b3d48a6  2018-01-06 16:30:00 UTC   \n",
       "371225  81b9f64b6f86312ce534fc4a17536a498b3d48a6  2018-01-06 19:10:00 UTC   \n",
       "371226  c6e64f267606c1a1b657786ba82cc328dc7722d6  2018-01-02 13:25:00 UTC   \n",
       "371227  4eb272b24b720ab73a573b5d6c69346a7ccac79f  2018-01-15 17:10:00 UTC   \n",
       "371228  4eb272b24b720ab73a573b5d6c69346a7ccac79f  2018-01-15 19:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          heat          hold               700                      705   \n",
       "3          heat          hold               705                      705   \n",
       "5          auto          hold               690                      740   \n",
       "7          heat          hold               703                      705   \n",
       "8          heat          auto               650                      745   \n",
       "...         ...           ...               ...                      ...   \n",
       "371224     heat          auto               734                      760   \n",
       "371225     heat          auto               764                      760   \n",
       "371226     heat          hold               758                      760   \n",
       "371227     heat          hold               748                      760   \n",
       "371228     heat          hold               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           705            CT      Waterford   \n",
       "3                           705            CT      Waterford   \n",
       "5                           682            CT    Barkhamsted   \n",
       "7                           705            CT      Waterford   \n",
       "8                           613            CT         Monroe   \n",
       "...                         ...           ...            ...   \n",
       "371224                      760            CT  East Hartford   \n",
       "371225                      760            CT  East Hartford   \n",
       "371226                      760            CT         Groton   \n",
       "371227                      760            CT      Greenwich   \n",
       "371228                      760            CT      Greenwich   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        25             False         False             True   \n",
       "3                        25             False         False             True   \n",
       "5                        30             False         False            False   \n",
       "7                        25             False         False             True   \n",
       "8                        40             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "371224                   50             False         False            False   \n",
       "371225                   50             False         False            False   \n",
       "371226                   66             False         False            False   \n",
       "371227                   10             False         False            False   \n",
       "371228                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                       Electric  \n",
       "3                       Electric  \n",
       "5                            Gas  \n",
       "7                       Electric  \n",
       "8                            Gas  \n",
       "...                          ...  \n",
       "371224                       Gas  \n",
       "371225                       Gas  \n",
       "371226                       Gas  \n",
       "371227                       Gas  \n",
       "371228                       Gas  \n",
       "\n",
       "[343575 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/CT/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/CT-day/2019-jan-day-CT.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c73238644b4d7d318233868c0fbae5a18eda1467</td>\n",
       "      <td>2019-01-30 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wallingford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4a8a1907ceccf4b74bd4173088eeeb88faf2a18b</td>\n",
       "      <td>2019-01-22 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7570fc772e83a3fe72dd8ba13bcc3e56410e40fb</td>\n",
       "      <td>2019-01-27 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>715</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwich</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4a8a1907ceccf4b74bd4173088eeeb88faf2a18b</td>\n",
       "      <td>2019-01-12 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>681</td>\n",
       "      <td>681</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c73238644b4d7d318233868c0fbae5a18eda1467</td>\n",
       "      <td>2019-01-24 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wallingford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546302</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2019-01-15 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546303</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2019-01-13 11:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546304</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2019-01-20 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546305</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2019-01-15 11:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>546306</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2019-01-07 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>493519 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c73238644b4d7d318233868c0fbae5a18eda1467  2019-01-30 15:30:00 UTC   \n",
       "1       4a8a1907ceccf4b74bd4173088eeeb88faf2a18b  2019-01-22 18:20:00 UTC   \n",
       "2       7570fc772e83a3fe72dd8ba13bcc3e56410e40fb  2019-01-27 16:45:00 UTC   \n",
       "3       4a8a1907ceccf4b74bd4173088eeeb88faf2a18b  2019-01-12 15:05:00 UTC   \n",
       "4       c73238644b4d7d318233868c0fbae5a18eda1467  2019-01-24 19:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "546302  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2019-01-15 12:00:00 UTC   \n",
       "546303  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2019-01-13 11:05:00 UTC   \n",
       "546304  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2019-01-20 19:00:00 UTC   \n",
       "546305  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2019-01-15 11:05:00 UTC   \n",
       "546306  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2019-01-07 14:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               695                      699   \n",
       "1          heat          hold               703                      701   \n",
       "2          auto          hold               669                      715   \n",
       "3          heat          hold               684                      681   \n",
       "4          heat          hold               712                      701   \n",
       "...         ...           ...               ...                      ...   \n",
       "546302     auto          hold               753                      820   \n",
       "546303     auto          auto               752                      820   \n",
       "546304     auto          hold               743                      820   \n",
       "546305     auto          hold               741                      820   \n",
       "546306     auto          hold               741                      820   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           699            CT  Wallingford   \n",
       "1                           701            CT      Windsor   \n",
       "2                           675            CT      Norwich   \n",
       "3                           681            CT      Windsor   \n",
       "4                           701            CT  Wallingford   \n",
       "...                         ...           ...          ...   \n",
       "546302                      760            CT   Ridgefield   \n",
       "546303                      760            CT   Ridgefield   \n",
       "546304                      760            CT   Ridgefield   \n",
       "546305                      760            CT   Ridgefield   \n",
       "546306                      760            CT   Ridgefield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "546302                   50             False         False            False   \n",
       "546303                   50             False         False            False   \n",
       "546304                   50             False         False            False   \n",
       "546305                   50             False         False            False   \n",
       "546306                   50             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "546302                       Gas  \n",
       "546303                       Gas  \n",
       "546304                       Gas  \n",
       "546305                       Gas  \n",
       "546306                       Gas  \n",
       "\n",
       "[493519 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/CT/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/CT-day/2020-jan-day-CT.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2020-01-28 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c2f1a84ed18ee42d39add99fe5e0e1c50fd288e3</td>\n",
       "      <td>2020-01-06 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>799</td>\n",
       "      <td>629</td>\n",
       "      <td>CT</td>\n",
       "      <td>Mystic</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1c0a777272f175574b611258b25e2b096fdac9df</td>\n",
       "      <td>2020-01-11 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>656</td>\n",
       "      <td>696</td>\n",
       "      <td>654</td>\n",
       "      <td>CT</td>\n",
       "      <td>Deep River</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4a8a1907ceccf4b74bd4173088eeeb88faf2a18b</td>\n",
       "      <td>2020-01-23 12:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>f4091c17f6d7dfa998f53a3e0159dbb214355c7f</td>\n",
       "      <td>2020-01-11 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544564</th>\n",
       "      <td>f8378dd5e363e435e2bb8f46ebd81e9e6f23af76</td>\n",
       "      <td>2020-01-30 07:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wilton</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544565</th>\n",
       "      <td>f8378dd5e363e435e2bb8f46ebd81e9e6f23af76</td>\n",
       "      <td>2020-01-30 08:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wilton</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544566</th>\n",
       "      <td>f8378dd5e363e435e2bb8f46ebd81e9e6f23af76</td>\n",
       "      <td>2020-01-30 08:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wilton</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544567</th>\n",
       "      <td>f8378dd5e363e435e2bb8f46ebd81e9e6f23af76</td>\n",
       "      <td>2020-01-30 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wilton</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>544568</th>\n",
       "      <td>b53165c97d6c052a1c950c8e26ac84eb9adc34b2</td>\n",
       "      <td>2020-01-02 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Bristol</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>480996 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "2       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2020-01-28 17:50:00 UTC   \n",
       "4       c2f1a84ed18ee42d39add99fe5e0e1c50fd288e3  2020-01-06 13:55:00 UTC   \n",
       "5       1c0a777272f175574b611258b25e2b096fdac9df  2020-01-11 16:35:00 UTC   \n",
       "6       4a8a1907ceccf4b74bd4173088eeeb88faf2a18b  2020-01-23 12:15:00 UTC   \n",
       "7       f4091c17f6d7dfa998f53a3e0159dbb214355c7f  2020-01-11 12:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "544564  f8378dd5e363e435e2bb8f46ebd81e9e6f23af76  2020-01-30 07:50:00 UTC   \n",
       "544565  f8378dd5e363e435e2bb8f46ebd81e9e6f23af76  2020-01-30 08:35:00 UTC   \n",
       "544566  f8378dd5e363e435e2bb8f46ebd81e9e6f23af76  2020-01-30 08:00:00 UTC   \n",
       "544567  f8378dd5e363e435e2bb8f46ebd81e9e6f23af76  2020-01-30 07:25:00 UTC   \n",
       "544568  b53165c97d6c052a1c950c8e26ac84eb9adc34b2  2020-01-02 18:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "2          heat          hold               707                      715   \n",
       "4          heat          hold               674                      799   \n",
       "5          heat          auto               656                      696   \n",
       "6          heat          hold               704                      685   \n",
       "7          heat          auto               731                      705   \n",
       "...         ...           ...               ...                      ...   \n",
       "544564     heat          auto               740                      780   \n",
       "544565     heat          auto               743                      780   \n",
       "544566     heat          auto               741                      780   \n",
       "544567     heat          auto               736                      780   \n",
       "544568     heat          auto               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "2                           715            CT     Windsor   \n",
       "4                           629            CT      Mystic   \n",
       "5                           654            CT  Deep River   \n",
       "6                           685            CT     Windsor   \n",
       "7                           705            CT     Enfield   \n",
       "...                         ...           ...         ...   \n",
       "544564                      760            CT      Wilton   \n",
       "544565                      760            CT      Wilton   \n",
       "544566                      760            CT      Wilton   \n",
       "544567                      760            CT      Wilton   \n",
       "544568                      760            CT     Bristol   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "2                        10             False         False            False   \n",
       "4                       116             False         False            False   \n",
       "5                       120             False         False            False   \n",
       "6                        10             False         False            False   \n",
       "7                        60             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "544564                   47             False         False            False   \n",
       "544565                   47             False         False            False   \n",
       "544566                   47             False         False            False   \n",
       "544567                   47             False         False            False   \n",
       "544568                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "544564                       Gas  \n",
       "544565                       Gas  \n",
       "544566                       Gas  \n",
       "544567                       Gas  \n",
       "544568                       Gas  \n",
       "\n",
       "[480996 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/CT/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/CT-day/2021-jan-day-CT.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3e7dd224b06e8259cc6c22ad07c6531900f383c3</td>\n",
       "      <td>2021-01-04 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>639</td>\n",
       "      <td>650</td>\n",
       "      <td>644</td>\n",
       "      <td>CT</td>\n",
       "      <td>cromwell</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>214f28562b2c6edb923553c2da33e34308417be2</td>\n",
       "      <td>2021-01-07 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>769</td>\n",
       "      <td>719</td>\n",
       "      <td>CT</td>\n",
       "      <td>Colchester</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ced1ff247136fb45b6f827deb5cb0c894d162471</td>\n",
       "      <td>2021-01-18 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>CT</td>\n",
       "      <td>Prospect</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b237ed0ef6f08e768c0cf6b7f91fbee3555fcd85</td>\n",
       "      <td>2021-01-08 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dd42deba452d2cea8e2ddb632972e864194a15b9</td>\n",
       "      <td>2021-01-21 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>643</td>\n",
       "      <td>650</td>\n",
       "      <td>643</td>\n",
       "      <td>CT</td>\n",
       "      <td>Madison</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383790</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-01-21 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383791</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-01-21 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383792</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-01-29 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383793</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-01-21 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383794</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-01-21 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>366185 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       3e7dd224b06e8259cc6c22ad07c6531900f383c3  2021-01-04 19:15:00 UTC   \n",
       "1       214f28562b2c6edb923553c2da33e34308417be2  2021-01-07 15:45:00 UTC   \n",
       "2       ced1ff247136fb45b6f827deb5cb0c894d162471  2021-01-18 17:30:00 UTC   \n",
       "3       b237ed0ef6f08e768c0cf6b7f91fbee3555fcd85  2021-01-08 14:25:00 UTC   \n",
       "4       dd42deba452d2cea8e2ddb632972e864194a15b9  2021-01-21 19:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "383790  251bf868c241659af914a268398fee21565c16c4  2021-01-21 15:00:00 UTC   \n",
       "383791  251bf868c241659af914a268398fee21565c16c4  2021-01-21 19:45:00 UTC   \n",
       "383792  251bf868c241659af914a268398fee21565c16c4  2021-01-29 14:55:00 UTC   \n",
       "383793  251bf868c241659af914a268398fee21565c16c4  2021-01-21 18:45:00 UTC   \n",
       "383794  251bf868c241659af914a268398fee21565c16c4  2021-01-21 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               639                      650   \n",
       "1          auto          hold               719                      769   \n",
       "2          heat          hold               703                      699   \n",
       "3          heat          hold               669                      677   \n",
       "4          heat          hold               643                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "383790     heat          hold               764                      760   \n",
       "383791     heat          hold               756                      760   \n",
       "383792     heat          hold               731                      760   \n",
       "383793     heat          hold               765                      760   \n",
       "383794     heat          hold               749                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           644            CT    cromwell   \n",
       "1                           719            CT  Colchester   \n",
       "2                           699            CT    Prospect   \n",
       "3                           677            CT     Enfield   \n",
       "4                           643            CT     Madison   \n",
       "...                         ...           ...         ...   \n",
       "383790                      760            CT   Naugatuck   \n",
       "383791                      760            CT   Naugatuck   \n",
       "383792                      760            CT   Naugatuck   \n",
       "383793                      760            CT   Naugatuck   \n",
       "383794                      760            CT   Naugatuck   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        65             False         False            False   \n",
       "3                        35              True         False            False   \n",
       "4                        50              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "383790                   30             False         False            False   \n",
       "383791                   30             False         False            False   \n",
       "383792                   30             False         False            False   \n",
       "383793                   30             False         False            False   \n",
       "383794                   30             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "383790                       Gas  \n",
       "383791                       Gas  \n",
       "383792                       Gas  \n",
       "383793                       Gas  \n",
       "383794                       Gas  \n",
       "\n",
       "[366185 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/CT/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01b5776ab24d14ace90a8296c49941a4ba8b0173</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>New MIlford</td>\n",
       "      <td>683.484731</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>685.509677</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05d474a41ce90b51ae9627f1dc2a5214d67b2741</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>630.937500</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>620.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0a751de00d076714c9f1c136f2203fae474b7ace</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>East Hampton</td>\n",
       "      <td>652.627729</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0b3c9b534cf69257d66a1102858406468dd61627</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>665.114754</td>\n",
       "      <td>665.004918</td>\n",
       "      <td>664.940984</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hamden</td>\n",
       "      <td>693.103448</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.462644</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>421</th>\n",
       "      <td>fc58d08fd7038c94b9cfcb3287a2bbb6e095745f</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>695.602740</td>\n",
       "      <td>749.904110</td>\n",
       "      <td>699.904110</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>422</th>\n",
       "      <td>fc92f3d7ab046b1e0482835f5cf10e18372dc434</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>700.048077</td>\n",
       "      <td>750.918269</td>\n",
       "      <td>697.716346</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>623.134771</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>611.832884</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>695.836999</td>\n",
       "      <td>692.338292</td>\n",
       "      <td>692.338292</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>ff535168246f1ad39e7eef2188305e629379579f</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Newington</td>\n",
       "      <td>693.601227</td>\n",
       "      <td>695.153374</td>\n",
       "      <td>695.153374</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2783 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01b5776ab24d14ace90a8296c49941a4ba8b0173   Jan  2017     auto   \n",
       "1    05d474a41ce90b51ae9627f1dc2a5214d67b2741   Jan  2017     heat   \n",
       "2    0a751de00d076714c9f1c136f2203fae474b7ace   Jan  2017     heat   \n",
       "3    0b3c9b534cf69257d66a1102858406468dd61627   Jan  2017     heat   \n",
       "4    0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf   Jan  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "421  fc58d08fd7038c94b9cfcb3287a2bbb6e095745f   Jan  2021     auto   \n",
       "422  fc92f3d7ab046b1e0482835f5cf10e18372dc434   Jan  2021     auto   \n",
       "423  fef94e274991646e0f28d91d1aaea3cd709d8825   Jan  2021     heat   \n",
       "424  ff498c11fdbb5d698084e0f03a29b69e76ee8566   Jan  2021     heat   \n",
       "425  ff535168246f1ad39e7eef2188305e629379579f   Jan  2021     heat   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            hold   New MIlford            683.484731   \n",
       "1            auto      Guilford            630.937500   \n",
       "2            auto  East Hampton            652.627729   \n",
       "3            auto     Greenwich            665.114754   \n",
       "4            auto        Hamden            693.103448   \n",
       "..            ...           ...                   ...   \n",
       "421          hold       Shelton            695.602740   \n",
       "422          hold     Fairfield            700.048077   \n",
       "423          hold    New Canaan            623.134771   \n",
       "424          hold   North Haven            695.836999   \n",
       "425          hold     Newington            693.601227   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     800.000000                   685.509677   \n",
       "1                     650.000000                   620.000000   \n",
       "2                     680.000000                   640.000000   \n",
       "3                     665.004918                   664.940984   \n",
       "4                     760.000000                   680.462644   \n",
       "..                           ...                          ...   \n",
       "421                   749.904110                   699.904110   \n",
       "422                   750.918269                   697.716346   \n",
       "423                   650.000000                   611.832884   \n",
       "424                   692.338292                   692.338292   \n",
       "425                   695.153374                   695.153374   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False            False  \n",
       "1                   35.0             False         False            False  \n",
       "2                    0.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                   25.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "421                 25.0             False         False            False  \n",
       "422                 40.0              True         False             True  \n",
       "423                  0.0             False         False            False  \n",
       "424                 70.0             False         False            False  \n",
       "425                 80.0             False         False            False  \n",
       "\n",
       "[2783 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/jan/\" + file)\n",
    "    CT_jan = pd.concat([CT_jan, df])\n",
    "    \n",
    "CT_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_jan.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/CT-day/2017-feb-day-CT.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ecc6ddba35a42fcb31e3c9649a554a96037d34b5</td>\n",
       "      <td>2017-02-06 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>650</td>\n",
       "      <td>631</td>\n",
       "      <td>CT</td>\n",
       "      <td>Redding</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>215c2dfd21213f343402901d6a5bee6dae1c5718</td>\n",
       "      <td>2017-02-11 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>780</td>\n",
       "      <td>657</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>90311ac9900b2ba8a03e5f6348c84a144db460b1</td>\n",
       "      <td>2017-02-04 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>638</td>\n",
       "      <td>676</td>\n",
       "      <td>644</td>\n",
       "      <td>CT</td>\n",
       "      <td>Mansfield</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>f1c7933114855923a9cf2d59719743f601e2da8c</td>\n",
       "      <td>2017-02-04 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>686</td>\n",
       "      <td>686</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwich</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>edddbdf7c078f31fcbb10678a197397afebd2601</td>\n",
       "      <td>2017-02-17 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>CT</td>\n",
       "      <td>Old Mystic</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113459</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2017-02-10 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113460</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2017-02-07 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113461</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2017-02-07 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113462</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2017-02-10 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113463</th>\n",
       "      <td>c6e64f267606c1a1b657786ba82cc328dc7722d6</td>\n",
       "      <td>2017-02-01 11:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Groton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97928 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "2       ecc6ddba35a42fcb31e3c9649a554a96037d34b5  2017-02-06 12:30:00 UTC   \n",
       "5       215c2dfd21213f343402901d6a5bee6dae1c5718  2017-02-11 19:15:00 UTC   \n",
       "6       90311ac9900b2ba8a03e5f6348c84a144db460b1  2017-02-04 19:10:00 UTC   \n",
       "8       f1c7933114855923a9cf2d59719743f601e2da8c  2017-02-04 17:40:00 UTC   \n",
       "9       edddbdf7c078f31fcbb10678a197397afebd2601  2017-02-17 17:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "113459  c6e64f267606c1a1b657786ba82cc328dc7722d6  2017-02-10 19:05:00 UTC   \n",
       "113460  c6e64f267606c1a1b657786ba82cc328dc7722d6  2017-02-07 16:00:00 UTC   \n",
       "113461  c6e64f267606c1a1b657786ba82cc328dc7722d6  2017-02-07 14:40:00 UTC   \n",
       "113462  c6e64f267606c1a1b657786ba82cc328dc7722d6  2017-02-10 18:00:00 UTC   \n",
       "113463  c6e64f267606c1a1b657786ba82cc328dc7722d6  2017-02-01 11:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "2          heat          hold               666                      650   \n",
       "5          auto          hold               676                      780   \n",
       "6          heat          hold               638                      676   \n",
       "8          heat          auto               699                      686   \n",
       "9          heat          hold               692                      665   \n",
       "...         ...           ...               ...                      ...   \n",
       "113459     heat          hold               758                      760   \n",
       "113460     heat          auto               761                      760   \n",
       "113461     heat          auto               755                      760   \n",
       "113462     heat          hold               758                      760   \n",
       "113463     heat          hold               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "2                           631            CT     Redding   \n",
       "5                           657            CT      Orange   \n",
       "6                           644            CT   Mansfield   \n",
       "8                           686            CT     Norwich   \n",
       "9                           665            CT  Old Mystic   \n",
       "...                         ...           ...         ...   \n",
       "113459                      760            CT      Groton   \n",
       "113460                      760            CT      Groton   \n",
       "113461                      760            CT      Groton   \n",
       "113462                      760            CT      Groton   \n",
       "113463                      760            CT      Groton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "2                        30              True         False            False   \n",
       "5                         0              True         False            False   \n",
       "6                        20              True         False             True   \n",
       "8                        45             False         False            False   \n",
       "9                        70             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "113459                   66             False         False            False   \n",
       "113460                   66             False         False            False   \n",
       "113461                   66             False         False            False   \n",
       "113462                   66             False         False            False   \n",
       "113463                   66             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "2                            Gas  \n",
       "5                            Gas  \n",
       "6                       Electric  \n",
       "8                            Gas  \n",
       "9                            Gas  \n",
       "...                          ...  \n",
       "113459                       Gas  \n",
       "113460                       Gas  \n",
       "113461                       Gas  \n",
       "113462                       Gas  \n",
       "113463                       Gas  \n",
       "\n",
       "[97928 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/CT/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/CT-day/2018-feb-day-CT.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2018-02-24 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>636</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2018-02-26 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>637</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2018-02-24 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>635</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>f4091c17f6d7dfa998f53a3e0159dbb214355c7f</td>\n",
       "      <td>2018-02-11 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>738</td>\n",
       "      <td>802</td>\n",
       "      <td>667</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>e62b302fe4554f54348b4c85776c7983d5d1ce5e</td>\n",
       "      <td>2018-02-27 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>CT</td>\n",
       "      <td>Farmington</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352385</th>\n",
       "      <td>e41a3c5004a67cafeafb255716f89a431046249d</td>\n",
       "      <td>2018-02-24 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352386</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2018-02-16 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352387</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2018-02-16 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352388</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2018-02-16 11:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352389</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2018-02-16 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>311499 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2018-02-24 15:00:00 UTC   \n",
       "3       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2018-02-26 19:45:00 UTC   \n",
       "5       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2018-02-24 11:40:00 UTC   \n",
       "8       f4091c17f6d7dfa998f53a3e0159dbb214355c7f  2018-02-11 15:35:00 UTC   \n",
       "9       e62b302fe4554f54348b4c85776c7983d5d1ce5e  2018-02-27 11:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "352385  e41a3c5004a67cafeafb255716f89a431046249d  2018-02-24 17:20:00 UTC   \n",
       "352386  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2018-02-16 16:15:00 UTC   \n",
       "352387  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2018-02-16 15:10:00 UTC   \n",
       "352388  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2018-02-16 11:10:00 UTC   \n",
       "352389  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2018-02-16 13:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               636                      685   \n",
       "3          auto          auto               637                      685   \n",
       "5          auto          auto               635                      685   \n",
       "8          heat          auto               738                      802   \n",
       "9          heat          hold               718                      716   \n",
       "...         ...           ...               ...                      ...   \n",
       "352385     heat          hold               761                      760   \n",
       "352386     heat          hold               757                      760   \n",
       "352387     heat          hold               756                      760   \n",
       "352388     heat          hold               760                      760   \n",
       "352389     heat          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           635            CT  Windsor Locks   \n",
       "3                           635            CT  Windsor Locks   \n",
       "5                           635            CT  Windsor Locks   \n",
       "8                           667            CT        Enfield   \n",
       "9                           716            CT     Farmington   \n",
       "...                         ...           ...            ...   \n",
       "352385                      760            CT      Greenwich   \n",
       "352386                      760            CT    New Milford   \n",
       "352387                      760            CT    New Milford   \n",
       "352388                      760            CT    New Milford   \n",
       "352389                      760            CT    New Milford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "5                        30             False         False            False   \n",
       "8                        60             False         False            False   \n",
       "9                         7             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "352385                   10             False         False            False   \n",
       "352386                   15             False         False            False   \n",
       "352387                   15             False         False            False   \n",
       "352388                   15             False         False            False   \n",
       "352389                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "8                            Gas  \n",
       "9                            Gas  \n",
       "...                          ...  \n",
       "352385                       Gas  \n",
       "352386                       Gas  \n",
       "352387                       Gas  \n",
       "352388                       Gas  \n",
       "352389                       Gas  \n",
       "\n",
       "[311499 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/CT/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/CT-day/2019-feb-day-CT.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c73238644b4d7d318233868c0fbae5a18eda1467</td>\n",
       "      <td>2019-02-04 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wallingford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c73238644b4d7d318233868c0fbae5a18eda1467</td>\n",
       "      <td>2019-02-01 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wallingford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7570fc772e83a3fe72dd8ba13bcc3e56410e40fb</td>\n",
       "      <td>2019-02-03 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>656</td>\n",
       "      <td>705</td>\n",
       "      <td>665</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwich</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c73238644b4d7d318233868c0fbae5a18eda1467</td>\n",
       "      <td>2019-02-15 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wallingford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2019-02-12 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>641</td>\n",
       "      <td>672</td>\n",
       "      <td>672</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367059</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2019-02-28 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367060</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2019-02-28 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367061</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2019-02-28 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367062</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2019-02-28 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367063</th>\n",
       "      <td>0d4871b896fc2a5b93007ca0a3a0799970fbf168</td>\n",
       "      <td>2019-02-28 11:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>New Milford</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>332530 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c73238644b4d7d318233868c0fbae5a18eda1467  2019-02-04 18:25:00 UTC   \n",
       "2       c73238644b4d7d318233868c0fbae5a18eda1467  2019-02-01 13:45:00 UTC   \n",
       "3       7570fc772e83a3fe72dd8ba13bcc3e56410e40fb  2019-02-03 13:30:00 UTC   \n",
       "4       c73238644b4d7d318233868c0fbae5a18eda1467  2019-02-15 18:40:00 UTC   \n",
       "5       528482e17661477da3fa31d0b3d158c6fb72e9aa  2019-02-12 13:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "367059  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2019-02-28 14:45:00 UTC   \n",
       "367060  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2019-02-28 13:45:00 UTC   \n",
       "367061  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2019-02-28 18:00:00 UTC   \n",
       "367062  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2019-02-28 17:55:00 UTC   \n",
       "367063  0d4871b896fc2a5b93007ca0a3a0799970fbf168  2019-02-28 11:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               719                      709   \n",
       "2          heat          hold               701                      699   \n",
       "3          auto          auto               656                      705   \n",
       "4          heat          hold               695                      699   \n",
       "5          heat          hold               641                      672   \n",
       "...         ...           ...               ...                      ...   \n",
       "367059     heat          hold               758                      760   \n",
       "367060     heat          hold               758                      760   \n",
       "367061     heat          hold               761                      760   \n",
       "367062     heat          hold               758                      760   \n",
       "367063     heat          hold               757                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           709            CT  Wallingford   \n",
       "2                           699            CT  Wallingford   \n",
       "3                           665            CT      Norwich   \n",
       "4                           699            CT  Wallingford   \n",
       "5                           672            CT    Woodstock   \n",
       "...                         ...           ...          ...   \n",
       "367059                      760            CT  New Milford   \n",
       "367060                      760            CT  New Milford   \n",
       "367061                      760            CT  New Milford   \n",
       "367062                      760            CT  New Milford   \n",
       "367063                      760            CT  New Milford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "367059                   15             False         False            False   \n",
       "367060                   15             False         False            False   \n",
       "367061                   15             False         False            False   \n",
       "367062                   15             False         False            False   \n",
       "367063                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "367059                       Gas  \n",
       "367060                       Gas  \n",
       "367061                       Gas  \n",
       "367062                       Gas  \n",
       "367063                       Gas  \n",
       "\n",
       "[332530 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/CT/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/CT-day/2020-feb-day-CT.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f4091c17f6d7dfa998f53a3e0159dbb214355c7f</td>\n",
       "      <td>2020-02-09 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>711</td>\n",
       "      <td>711</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b5fdea0595d25b4775cbef1b9dac1f2857940c7d</td>\n",
       "      <td>2020-02-11 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>656</td>\n",
       "      <td>780</td>\n",
       "      <td>627</td>\n",
       "      <td>CT</td>\n",
       "      <td>Tolland</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a8bf398ef2aaff7fe69277442a23274c472e658e</td>\n",
       "      <td>2020-02-18 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>CT</td>\n",
       "      <td>East haven</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>b9eb493652b04bdbd9a25c27c17a34f3e00646cd</td>\n",
       "      <td>2020-02-14 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>770</td>\n",
       "      <td>708</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>105d4bcd5c19ff6242a31be48b095281b1f41eec</td>\n",
       "      <td>2020-02-15 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500242</th>\n",
       "      <td>764deab9260532acb2bb9a882b120eb77973625c</td>\n",
       "      <td>2020-02-18 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500243</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2020-02-15 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500244</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2020-02-15 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500245</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2020-02-15 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500246</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2020-02-12 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>436913 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f4091c17f6d7dfa998f53a3e0159dbb214355c7f  2020-02-09 14:00:00 UTC   \n",
       "2       b5fdea0595d25b4775cbef1b9dac1f2857940c7d  2020-02-11 12:55:00 UTC   \n",
       "4       a8bf398ef2aaff7fe69277442a23274c472e658e  2020-02-18 18:50:00 UTC   \n",
       "11      b9eb493652b04bdbd9a25c27c17a34f3e00646cd  2020-02-14 15:00:00 UTC   \n",
       "12      105d4bcd5c19ff6242a31be48b095281b1f41eec  2020-02-15 18:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "500242  764deab9260532acb2bb9a882b120eb77973625c  2020-02-18 19:15:00 UTC   \n",
       "500243  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2020-02-15 19:40:00 UTC   \n",
       "500244  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2020-02-15 18:40:00 UTC   \n",
       "500245  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2020-02-15 18:55:00 UTC   \n",
       "500246  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2020-02-12 12:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               710                      711   \n",
       "2          auto          auto               656                      780   \n",
       "4          heat          hold               708                      702   \n",
       "11         heat          auto               691                      770   \n",
       "12         heat          hold               702                      705   \n",
       "...         ...           ...               ...                      ...   \n",
       "500242     auto          hold               723                      820   \n",
       "500243     auto          hold               751                      810   \n",
       "500244     auto          hold               747                      810   \n",
       "500245     auto          hold               761                      810   \n",
       "500246     auto          hold               733                      810   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           711            CT     Enfield   \n",
       "2                           627            CT     Tolland   \n",
       "4                           702            CT  East haven   \n",
       "11                          708            CT      Orange   \n",
       "12                          705            CT   Watertown   \n",
       "...                         ...           ...         ...   \n",
       "500242                      760            CT   Fairfield   \n",
       "500243                      760            CT  Ridgefield   \n",
       "500244                      760            CT  Ridgefield   \n",
       "500245                      760            CT  Ridgefield   \n",
       "500246                      760            CT  Ridgefield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60             False         False            False   \n",
       "2                        25             False         False            False   \n",
       "4                        65             False         False            False   \n",
       "11                       55             False         False            False   \n",
       "12                       25              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "500242                    0              True         False            False   \n",
       "500243                   50             False         False            False   \n",
       "500244                   50             False         False            False   \n",
       "500245                   50             False         False            False   \n",
       "500246                   50             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "11                           Gas  \n",
       "12                           Gas  \n",
       "...                          ...  \n",
       "500242                       Gas  \n",
       "500243                       Gas  \n",
       "500244                       Gas  \n",
       "500245                       Gas  \n",
       "500246                       Gas  \n",
       "\n",
       "[436913 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/CT/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/CT-day/2021-feb-day-CT.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ced1ff247136fb45b6f827deb5cb0c894d162471</td>\n",
       "      <td>2021-02-15 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>CT</td>\n",
       "      <td>Prospect</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-02-10 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-02-22 11:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-02-01 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>748</td>\n",
       "      <td>748</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-02-20 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350768</th>\n",
       "      <td>251bf868c241659af914a268398fee21565c16c4</td>\n",
       "      <td>2021-02-23 08:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Naugatuck</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350769</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2021-02-26 09:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350770</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2021-02-26 09:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350771</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2021-02-26 08:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350772</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2021-02-26 08:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>326382 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       ced1ff247136fb45b6f827deb5cb0c894d162471  2021-02-15 17:55:00 UTC   \n",
       "1       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-02-10 18:55:00 UTC   \n",
       "2       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-02-22 11:40:00 UTC   \n",
       "3       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-02-01 14:40:00 UTC   \n",
       "4       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-02-20 15:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "350768  251bf868c241659af914a268398fee21565c16c4  2021-02-23 08:45:00 UTC   \n",
       "350769  d844acbe1727e39f17d98fc6505c5afb592cb458  2021-02-26 09:10:00 UTC   \n",
       "350770  d844acbe1727e39f17d98fc6505c5afb592cb458  2021-02-26 09:00:00 UTC   \n",
       "350771  d844acbe1727e39f17d98fc6505c5afb592cb458  2021-02-26 08:05:00 UTC   \n",
       "350772  d844acbe1727e39f17d98fc6505c5afb592cb458  2021-02-26 08:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               699                      702   \n",
       "1          heat          hold               730                      735   \n",
       "2          heat          hold               723                      725   \n",
       "3          heat          hold               724                      748   \n",
       "4          heat          hold               710                      715   \n",
       "...         ...           ...               ...                      ...   \n",
       "350768     heat          hold               750                      760   \n",
       "350769     heat          hold               746                      760   \n",
       "350770     heat          hold               760                      760   \n",
       "350771     heat          hold               752                      760   \n",
       "350772     heat          hold               754                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           702            CT    Prospect   \n",
       "1                           735            CT     Windsor   \n",
       "2                           725            CT     Windsor   \n",
       "3                           748            CT     Windsor   \n",
       "4                           715            CT     Windsor   \n",
       "...                         ...           ...         ...   \n",
       "350768                      760            CT   Naugatuck   \n",
       "350769                      760            CT  Torrington   \n",
       "350770                      760            CT  Torrington   \n",
       "350771                      760            CT  Torrington   \n",
       "350772                      760            CT  Torrington   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        65             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "350768                   30             False         False            False   \n",
       "350769                   15             False         False            False   \n",
       "350770                   15             False         False            False   \n",
       "350771                   15             False         False            False   \n",
       "350772                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "350768                       Gas  \n",
       "350769                       Gas  \n",
       "350770                       Gas  \n",
       "350771                       Gas  \n",
       "350772                       Gas  \n",
       "\n",
       "[326382 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/CT/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01b5776ab24d14ace90a8296c49941a4ba8b0173</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>New MIlford</td>\n",
       "      <td>688.446360</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05d474a41ce90b51ae9627f1dc2a5214d67b2741</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>689.846154</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0a751de00d076714c9f1c136f2203fae474b7ace</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>East Hampton</td>\n",
       "      <td>646.242312</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>638.308733</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0b3c9b534cf69257d66a1102858406468dd61627</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>669.248031</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hamden</td>\n",
       "      <td>701.683333</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.491667</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>fc58d08fd7038c94b9cfcb3287a2bbb6e095745f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>694.914634</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>fc92f3d7ab046b1e0482835f5cf10e18372dc434</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>674.650000</td>\n",
       "      <td>699.900000</td>\n",
       "      <td>699.316667</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>413</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>613.409683</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>610.355680</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>414</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>693.165198</td>\n",
       "      <td>689.720264</td>\n",
       "      <td>689.720264</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415</th>\n",
       "      <td>ff535168246f1ad39e7eef2188305e629379579f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Newington</td>\n",
       "      <td>698.011364</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2553 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01b5776ab24d14ace90a8296c49941a4ba8b0173   feb  2017     auto   \n",
       "1    05d474a41ce90b51ae9627f1dc2a5214d67b2741   feb  2017     heat   \n",
       "2    0a751de00d076714c9f1c136f2203fae474b7ace   feb  2017     heat   \n",
       "3    0b3c9b534cf69257d66a1102858406468dd61627   feb  2017     heat   \n",
       "4    0c0744b5e162dc1ad0a5e47d53e8650ba8ade8cf   feb  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "411  fc58d08fd7038c94b9cfcb3287a2bbb6e095745f   feb  2021     auto   \n",
       "412  fc92f3d7ab046b1e0482835f5cf10e18372dc434   feb  2021     heat   \n",
       "413  fef94e274991646e0f28d91d1aaea3cd709d8825   feb  2021     heat   \n",
       "414  ff498c11fdbb5d698084e0f03a29b69e76ee8566   feb  2021     heat   \n",
       "415  ff535168246f1ad39e7eef2188305e629379579f   feb  2021     heat   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            hold   New MIlford            688.446360   \n",
       "1            hold      Guilford            689.846154   \n",
       "2            auto  East Hampton            646.242312   \n",
       "3            auto     Greenwich            669.248031   \n",
       "4            auto        Hamden            701.683333   \n",
       "..            ...           ...                   ...   \n",
       "411          hold       Shelton            694.914634   \n",
       "412          hold     Fairfield            674.650000   \n",
       "413          hold    New Canaan            613.409683   \n",
       "414          hold   North Haven            693.165198   \n",
       "415          hold     Newington            698.011364   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     800.000000                   690.000000   \n",
       "1                     695.000000                   695.000000   \n",
       "2                     680.000000                   638.308733   \n",
       "3                     670.000000                   670.000000   \n",
       "4                     760.000000                   680.491667   \n",
       "..                           ...                          ...   \n",
       "411                   750.000000                   700.000000   \n",
       "412                   699.900000                   699.316667   \n",
       "413                   650.000000                   610.355680   \n",
       "414                   689.720264                   689.720264   \n",
       "415                   700.000000                   700.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False            False  \n",
       "1                   35.0             False         False            False  \n",
       "2                    0.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                   25.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "411                 25.0             False         False            False  \n",
       "412                 40.0              True         False             True  \n",
       "413                  0.0             False         False            False  \n",
       "414                 70.0             False         False            False  \n",
       "415                 80.0             False         False            False  \n",
       "\n",
       "[2553 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/feb/\" + file)\n",
    "    CT_feb = pd.concat([CT_feb, df])\n",
    "    \n",
    "CT_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_feb.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/CT-day/2017-jun-day-CT.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a83e1ff380a8949f1ca87840e41db535065e5120</td>\n",
       "      <td>2017-06-16 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Suffield</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a83e1ff380a8949f1ca87840e41db535065e5120</td>\n",
       "      <td>2017-06-11 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Suffield</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>565c7e8b066f34db999e142b69b61c1a6c189829</td>\n",
       "      <td>2017-06-22 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>783</td>\n",
       "      <td>817</td>\n",
       "      <td>662</td>\n",
       "      <td>CT</td>\n",
       "      <td>Killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b237ed0ef6f08e768c0cf6b7f91fbee3555fcd85</td>\n",
       "      <td>2017-06-13 09:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a83e1ff380a8949f1ca87840e41db535065e5120</td>\n",
       "      <td>2017-06-14 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Suffield</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170851</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-06-02 11:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170852</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-06-02 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170853</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-06-02 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170854</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-06-23 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170855</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-06-20 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>158989 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a83e1ff380a8949f1ca87840e41db535065e5120  2017-06-16 17:30:00 UTC   \n",
       "1       a83e1ff380a8949f1ca87840e41db535065e5120  2017-06-11 14:45:00 UTC   \n",
       "2       565c7e8b066f34db999e142b69b61c1a6c189829  2017-06-22 16:10:00 UTC   \n",
       "3       b237ed0ef6f08e768c0cf6b7f91fbee3555fcd85  2017-06-13 09:05:00 UTC   \n",
       "4       a83e1ff380a8949f1ca87840e41db535065e5120  2017-06-14 19:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "170851  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-06-02 11:25:00 UTC   \n",
       "170852  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-06-02 14:10:00 UTC   \n",
       "170853  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-06-02 13:25:00 UTC   \n",
       "170854  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-06-23 18:30:00 UTC   \n",
       "170855  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-06-20 17:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               739                      735   \n",
       "1          auto          auto               737                      735   \n",
       "2          cool          hold               783                      817   \n",
       "3          auto          auto               729                      725   \n",
       "4          auto          auto               739                      735   \n",
       "...         ...           ...               ...                      ...   \n",
       "170851     cool          hold               698                      760   \n",
       "170852     cool          hold               711                      760   \n",
       "170853     cool          hold               709                      760   \n",
       "170854     cool          hold               764                      760   \n",
       "170855     cool          hold               756                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           685            CT      Suffield   \n",
       "1                           685            CT      Suffield   \n",
       "2                           662            CT  Killingworth   \n",
       "3                           675            CT       Enfield   \n",
       "4                           685            CT      Suffield   \n",
       "...                         ...           ...           ...   \n",
       "170851                      760            CT  wethersfield   \n",
       "170852                      760            CT  wethersfield   \n",
       "170853                      760            CT  wethersfield   \n",
       "170854                      760            CT  wethersfield   \n",
       "170855                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        15             False         False            False   \n",
       "3                        35              True         False            False   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "170851                  100             False         False             True   \n",
       "170852                  100             False         False             True   \n",
       "170853                  100             False         False             True   \n",
       "170854                  100             False         False             True   \n",
       "170855                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "170851                  Electric  \n",
       "170852                  Electric  \n",
       "170853                  Electric  \n",
       "170854                  Electric  \n",
       "170855                  Electric  \n",
       "\n",
       "[158989 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/CT/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/CT-day/2018-jun-day-CT.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-06-20 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>782</td>\n",
       "      <td>782</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-06-26 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-06-01 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>692</td>\n",
       "      <td>632</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-06-29 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-06-18 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325820</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-06-22 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325821</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-06-25 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325822</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-06-27 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325823</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-06-17 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325824</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-06-21 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>298226 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-06-20 13:30:00 UTC   \n",
       "1       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-06-26 17:10:00 UTC   \n",
       "2       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-06-01 17:15:00 UTC   \n",
       "3       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-06-29 15:55:00 UTC   \n",
       "4       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-06-18 17:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "325820  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-06-22 16:30:00 UTC   \n",
       "325821  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-06-25 17:20:00 UTC   \n",
       "325822  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-06-27 12:05:00 UTC   \n",
       "325823  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-06-17 19:25:00 UTC   \n",
       "325824  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-06-21 19:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               699                      782   \n",
       "1          auto          auto               710                      725   \n",
       "2          auto          hold               691                      692   \n",
       "3          cool          hold               703                      702   \n",
       "4          cool          hold               717                      742   \n",
       "...         ...           ...               ...                      ...   \n",
       "325820     cool          hold               732                      760   \n",
       "325821     cool          hold               739                      760   \n",
       "325822     cool          auto               707                      760   \n",
       "325823     cool          hold               743                      760   \n",
       "325824     cool          hold               752                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           782            CT     Woodstock   \n",
       "1                           675            CT   Broad Brook   \n",
       "2                           632            CT     Woodstock   \n",
       "3                           702            CT     Woodstock   \n",
       "4                           742            CT     Woodstock   \n",
       "...                         ...           ...           ...   \n",
       "325820                      760            CT  wethersfield   \n",
       "325821                      760            CT  wethersfield   \n",
       "325822                      760            CT  wethersfield   \n",
       "325823                      760            CT  wethersfield   \n",
       "325824                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        60             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "325820                  100             False         False             True   \n",
       "325821                  100             False         False             True   \n",
       "325822                  100             False         False             True   \n",
       "325823                  100             False         False             True   \n",
       "325824                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "325820                  Electric  \n",
       "325821                  Electric  \n",
       "325822                  Electric  \n",
       "325823                  Electric  \n",
       "325824                  Electric  \n",
       "\n",
       "[298226 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/CT/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/CT-day/2019-jun-day-CT.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>564c6aec11077a013c08daef6f741cea54a99227</td>\n",
       "      <td>2019-06-02 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Southbury</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>564c6aec11077a013c08daef6f741cea54a99227</td>\n",
       "      <td>2019-06-14 10:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>745</td>\n",
       "      <td>732</td>\n",
       "      <td>CT</td>\n",
       "      <td>Southbury</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-06-20 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2019-06-01 11:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f84fcb0cde327263283f23cea32fdd90e0e0d48e</td>\n",
       "      <td>2019-06-29 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>724</td>\n",
       "      <td>724</td>\n",
       "      <td>CT</td>\n",
       "      <td>Wethersfield</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416637</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-06-29 10:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416638</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-06-29 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416639</th>\n",
       "      <td>e84cfd4b5dfd576f4ba539b81c92c56b6584367a</td>\n",
       "      <td>2019-06-29 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>North Grosvenordale</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416640</th>\n",
       "      <td>e84cfd4b5dfd576f4ba539b81c92c56b6584367a</td>\n",
       "      <td>2019-06-29 11:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>North Grosvenordale</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416641</th>\n",
       "      <td>e84cfd4b5dfd576f4ba539b81c92c56b6584367a</td>\n",
       "      <td>2019-06-29 11:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>North Grosvenordale</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>369257 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       564c6aec11077a013c08daef6f741cea54a99227  2019-06-02 12:55:00 UTC   \n",
       "1       564c6aec11077a013c08daef6f741cea54a99227  2019-06-14 10:00:00 UTC   \n",
       "2       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-06-20 17:20:00 UTC   \n",
       "3       e9870d26f289a344a43ab52d729cb869f3baaa62  2019-06-01 11:20:00 UTC   \n",
       "4       f84fcb0cde327263283f23cea32fdd90e0e0d48e  2019-06-29 17:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "416637  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-06-29 10:15:00 UTC   \n",
       "416638  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-06-29 19:35:00 UTC   \n",
       "416639  e84cfd4b5dfd576f4ba539b81c92c56b6584367a  2019-06-29 11:00:00 UTC   \n",
       "416640  e84cfd4b5dfd576f4ba539b81c92c56b6584367a  2019-06-29 11:05:00 UTC   \n",
       "416641  e84cfd4b5dfd576f4ba539b81c92c56b6584367a  2019-06-29 11:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               684                      685   \n",
       "1          cool          hold               678                      745   \n",
       "2          cool          hold               708                      705   \n",
       "3          auto          auto               694                      725   \n",
       "4          cool          hold               726                      724   \n",
       "...         ...           ...               ...                      ...   \n",
       "416637     cool          hold               762                      760   \n",
       "416638     cool          hold               763                      760   \n",
       "416639     cool          hold               732                      760   \n",
       "416640     cool          hold               732                      760   \n",
       "416641     cool          hold               732                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                           685            CT            Southbury   \n",
       "1                           732            CT            Southbury   \n",
       "2                           705            CT            Waterford   \n",
       "3                           675            CT          Broad Brook   \n",
       "4                           724            CT         Wethersfield   \n",
       "...                         ...           ...                  ...   \n",
       "416637                      760            CT        West Hartford   \n",
       "416638                      760            CT        West Hartford   \n",
       "416639                      760            CT  North Grosvenordale   \n",
       "416640                      760            CT  North Grosvenordale   \n",
       "416641                      760            CT  North Grosvenordale   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "1                        40             False         False            False   \n",
       "2                        25             False         False             True   \n",
       "3                        60             False         False            False   \n",
       "4                        20              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "416637                   75              True         False            False   \n",
       "416638                   75              True         False            False   \n",
       "416639                   80             False         False            False   \n",
       "416640                   80             False         False            False   \n",
       "416641                   80             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "416637                       Gas  \n",
       "416638                       Gas  \n",
       "416639                       Gas  \n",
       "416640                       Gas  \n",
       "416641                       Gas  \n",
       "\n",
       "[369257 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/CT/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/CT-day/2020-jun-day-CT.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0090bb6fc40c84b7f42c125689e22350e6128e3e</td>\n",
       "      <td>2020-06-03 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>740</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2020-06-02 11:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>685</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b9eb493652b04bdbd9a25c27c17a34f3e00646cd</td>\n",
       "      <td>2020-06-30 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>730</td>\n",
       "      <td>664</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0090bb6fc40c84b7f42c125689e22350e6128e3e</td>\n",
       "      <td>2020-06-24 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>750</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>edd88b332aea2fee543048ec1b27d65c159e4bab</td>\n",
       "      <td>2020-06-01 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Newtown</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478213</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2020-06-06 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478214</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2020-06-06 11:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478215</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2020-06-06 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478216</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2020-06-06 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478217</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2020-06-06 08:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>455645 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0090bb6fc40c84b7f42c125689e22350e6128e3e  2020-06-03 17:25:00 UTC   \n",
       "1       e9870d26f289a344a43ab52d729cb869f3baaa62  2020-06-02 11:10:00 UTC   \n",
       "2       b9eb493652b04bdbd9a25c27c17a34f3e00646cd  2020-06-30 19:55:00 UTC   \n",
       "3       0090bb6fc40c84b7f42c125689e22350e6128e3e  2020-06-24 15:45:00 UTC   \n",
       "4       edd88b332aea2fee543048ec1b27d65c159e4bab  2020-06-01 18:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "478213  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2020-06-06 17:25:00 UTC   \n",
       "478214  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2020-06-06 11:15:00 UTC   \n",
       "478215  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2020-06-06 16:20:00 UTC   \n",
       "478216  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2020-06-06 12:15:00 UTC   \n",
       "478217  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2020-06-06 08:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               732                      740   \n",
       "1          auto          auto               685                      735   \n",
       "2          cool          auto               732                      730   \n",
       "3          cool          hold               749                      750   \n",
       "4          heat          hold               710                      715   \n",
       "...         ...           ...               ...                      ...   \n",
       "478213     cool          hold               762                      760   \n",
       "478214     cool          auto               762                      760   \n",
       "478215     cool          hold               758                      760   \n",
       "478216     cool          auto               760                      760   \n",
       "478217     cool          auto               762                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           731            CT         Monroe   \n",
       "1                           685            CT    Broad Brook   \n",
       "2                           664            CT         Orange   \n",
       "3                           731            CT         Monroe   \n",
       "4                           715            CT        Newtown   \n",
       "...                         ...           ...            ...   \n",
       "478213                      760            CT  West Hartford   \n",
       "478214                      760            CT  West Hartford   \n",
       "478215                      760            CT  West Hartford   \n",
       "478216                      760            CT  West Hartford   \n",
       "478217                      760            CT  West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        60             False         False            False   \n",
       "2                        55             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "478213                   75              True         False            False   \n",
       "478214                   75              True         False            False   \n",
       "478215                   75              True         False            False   \n",
       "478216                   75              True         False            False   \n",
       "478217                   75              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "478213                       Gas  \n",
       "478214                       Gas  \n",
       "478215                       Gas  \n",
       "478216                       Gas  \n",
       "478217                       Gas  \n",
       "\n",
       "[455645 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/CT/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/CT-day/2021-jun-day-CT.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>71b7a96cb3b5e2af708eb1e817f3e24a0023cb51</td>\n",
       "      <td>2021-06-25 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>723</td>\n",
       "      <td>723</td>\n",
       "      <td>CT</td>\n",
       "      <td>Farmington</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-06-23 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>715</td>\n",
       "      <td>665</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bd36ac1884d1b47040031d82aecd4533604135d8</td>\n",
       "      <td>2021-06-22 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>650</td>\n",
       "      <td>642</td>\n",
       "      <td>CT</td>\n",
       "      <td>Tolland</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0fa0b4c084347c87c094d0ff966f45f13fa5b251</td>\n",
       "      <td>2021-06-18 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>762</td>\n",
       "      <td>756</td>\n",
       "      <td>CT</td>\n",
       "      <td>Hartford</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2021-06-12 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>715</td>\n",
       "      <td>665</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309764</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-06-04 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309765</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-06-05 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309766</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-06-05 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309767</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-06-05 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309768</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-06-04 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>291762 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       71b7a96cb3b5e2af708eb1e817f3e24a0023cb51  2021-06-25 11:35:00 UTC   \n",
       "1       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-06-23 18:35:00 UTC   \n",
       "2       bd36ac1884d1b47040031d82aecd4533604135d8  2021-06-22 17:20:00 UTC   \n",
       "3       0fa0b4c084347c87c094d0ff966f45f13fa5b251  2021-06-18 19:40:00 UTC   \n",
       "4       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2021-06-12 17:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "309764  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-06-04 18:05:00 UTC   \n",
       "309765  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-06-05 18:50:00 UTC   \n",
       "309766  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-06-05 18:30:00 UTC   \n",
       "309767  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-06-05 19:30:00 UTC   \n",
       "309768  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-06-04 19:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               721                      723   \n",
       "1          cool          hold               718                      715   \n",
       "2          heat          hold               674                      650   \n",
       "3          cool          hold               768                      762   \n",
       "4          cool          hold               760                      715   \n",
       "...         ...           ...               ...                      ...   \n",
       "309764     cool          hold               729                      760   \n",
       "309765     cool          hold               729                      760   \n",
       "309766     cool          hold               726                      760   \n",
       "309767     cool          hold               735                      760   \n",
       "309768     cool          hold               732                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           723            CT    Farmington   \n",
       "1                           665            CT       Windsor   \n",
       "2                           642            CT       Tolland   \n",
       "3                           756            CT      Hartford   \n",
       "4                           665            CT       Windsor   \n",
       "...                         ...           ...           ...   \n",
       "309764                      760            CT  wethersfield   \n",
       "309765                      760            CT  wethersfield   \n",
       "309766                      760            CT  wethersfield   \n",
       "309767                      760            CT  wethersfield   \n",
       "309768                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20              True         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        29             False         False            False   \n",
       "3                        90             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "309764                  100             False         False             True   \n",
       "309765                  100             False         False             True   \n",
       "309766                  100             False         False             True   \n",
       "309767                  100             False         False             True   \n",
       "309768                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "309764                  Electric  \n",
       "309765                  Electric  \n",
       "309766                  Electric  \n",
       "309767                  Electric  \n",
       "309768                  Electric  \n",
       "\n",
       "[291762 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/CT/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>007f9b37f1ac5cfe9806f9a1a47809855e7e31b8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Windsor</td>\n",
       "      <td>717.464162</td>\n",
       "      <td>732.698266</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>739.454545</td>\n",
       "      <td>709.272727</td>\n",
       "      <td>692.727273</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>765.000000</td>\n",
       "      <td>699.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>699.173669</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>686.504202</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>708.696148</td>\n",
       "      <td>708.810271</td>\n",
       "      <td>708.770328</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>446</th>\n",
       "      <td>fc58d08fd7038c94b9cfcb3287a2bbb6e095745f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>711.339921</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>fc92f3d7ab046b1e0482835f5cf10e18372dc434</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>694.968354</td>\n",
       "      <td>687.664557</td>\n",
       "      <td>687.145570</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>650.846603</td>\n",
       "      <td>647.909746</td>\n",
       "      <td>647.802334</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>715.425000</td>\n",
       "      <td>722.812500</td>\n",
       "      <td>722.812500</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>699.350000</td>\n",
       "      <td>699.500000</td>\n",
       "      <td>699.500000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2743 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    007f9b37f1ac5cfe9806f9a1a47809855e7e31b8   jun  2017     cool   \n",
       "1    0094e2c467552f4b15765889040931605fb22890   jun  2017     cool   \n",
       "2    0094e2c467552f4b15765889040931605fb22890   jun  2017     cool   \n",
       "3    0382f14f425de2d9f9645def17ee3cfeada55f78   jun  2017     cool   \n",
       "4    0382f14f425de2d9f9645def17ee3cfeada55f78   jun  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "446  fc58d08fd7038c94b9cfcb3287a2bbb6e095745f   jun  2021     cool   \n",
       "447  fc92f3d7ab046b1e0482835f5cf10e18372dc434   jun  2021     cool   \n",
       "448  fef94e274991646e0f28d91d1aaea3cd709d8825   jun  2021     cool   \n",
       "449  ff498c11fdbb5d698084e0f03a29b69e76ee8566   jun  2021     cool   \n",
       "450  ff498c11fdbb5d698084e0f03a29b69e76ee8566   jun  2021     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold  South Windsor            717.464162   \n",
       "1            auto        Norwalk            739.454545   \n",
       "2            hold        Norwalk            765.000000   \n",
       "3            auto      Waterford            699.173669   \n",
       "4            hold      Waterford            708.696148   \n",
       "..            ...            ...                   ...   \n",
       "446          hold        Shelton            711.339921   \n",
       "447          hold      Fairfield            694.968354   \n",
       "448          hold     New Canaan            650.846603   \n",
       "449          hold    North Haven            715.425000   \n",
       "450          hold    North Haven            699.350000   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     732.698266                   660.000000   \n",
       "1                     709.272727                   692.727273   \n",
       "2                     699.000000                   700.000000   \n",
       "3                     700.000000                   686.504202   \n",
       "4                     708.810271                   708.770328   \n",
       "..                           ...                          ...   \n",
       "446                   710.000000                   710.000000   \n",
       "447                   687.664557                   687.145570   \n",
       "448                   647.909746                   647.802334   \n",
       "449                   722.812500                   722.812500   \n",
       "450                   699.500000                   699.500000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   27.0             False         False            False  \n",
       "2                   27.0             False         False            False  \n",
       "3                   25.0             False         False             True  \n",
       "4                   25.0             False         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "446                 25.0             False         False            False  \n",
       "447                 40.0              True         False             True  \n",
       "448                  0.0             False         False            False  \n",
       "449                 70.0             False         False            False  \n",
       "450                 70.0             False         False            False  \n",
       "\n",
       "[2743 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/jun/\" + file)\n",
    "    CT_jun = pd.concat([CT_jun, df])\n",
    "    \n",
    "CT_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_jun.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/CT-day/2017-jul-day-CT.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e32eeab5de758aca829b12a6c53604bdbeba9734</td>\n",
       "      <td>2017-07-01 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>747</td>\n",
       "      <td>747</td>\n",
       "      <td>CT</td>\n",
       "      <td>Litchfield</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bb815c20941dfc2c09da049e6d22845484811dd3</td>\n",
       "      <td>2017-07-16 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>748</td>\n",
       "      <td>667</td>\n",
       "      <td>CT</td>\n",
       "      <td>Easton</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7570fc772e83a3fe72dd8ba13bcc3e56410e40fb</td>\n",
       "      <td>2017-07-29 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>725</td>\n",
       "      <td>638</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwich</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>582e46f241d8ee21335889156509339ff2e12628</td>\n",
       "      <td>2017-07-28 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>772</td>\n",
       "      <td>708</td>\n",
       "      <td>CT</td>\n",
       "      <td>Killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>565c7e8b066f34db999e142b69b61c1a6c189829</td>\n",
       "      <td>2017-07-16 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>780</td>\n",
       "      <td>772</td>\n",
       "      <td>CT</td>\n",
       "      <td>Killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226220</th>\n",
       "      <td>7f429ed3e777b71b73c74afb5d188105933ebab4</td>\n",
       "      <td>2017-07-23 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>East Hampton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226221</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-07-05 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226222</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-07-12 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226223</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-07-23 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226224</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-07-08 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217112 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e32eeab5de758aca829b12a6c53604bdbeba9734  2017-07-01 15:55:00 UTC   \n",
       "1       bb815c20941dfc2c09da049e6d22845484811dd3  2017-07-16 16:45:00 UTC   \n",
       "2       7570fc772e83a3fe72dd8ba13bcc3e56410e40fb  2017-07-29 15:00:00 UTC   \n",
       "3       582e46f241d8ee21335889156509339ff2e12628  2017-07-28 15:35:00 UTC   \n",
       "4       565c7e8b066f34db999e142b69b61c1a6c189829  2017-07-16 12:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "226220  7f429ed3e777b71b73c74afb5d188105933ebab4  2017-07-23 15:15:00 UTC   \n",
       "226221  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-07-05 17:35:00 UTC   \n",
       "226222  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-07-12 16:40:00 UTC   \n",
       "226223  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-07-23 19:00:00 UTC   \n",
       "226224  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-07-08 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               751                      747   \n",
       "1          cool          auto               724                      748   \n",
       "2          cool          auto               703                      725   \n",
       "3          cool          auto               733                      772   \n",
       "4          cool          auto               717                      780   \n",
       "...         ...           ...               ...                      ...   \n",
       "226220     cool          hold               760                      760   \n",
       "226221     cool          hold               743                      760   \n",
       "226222     cool          hold               754                      760   \n",
       "226223     cool          hold               743                      760   \n",
       "226224     cool          hold               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           747            CT    Litchfield   \n",
       "1                           667            CT        Easton   \n",
       "2                           638            CT       Norwich   \n",
       "3                           708            CT  Killingworth   \n",
       "4                           772            CT  Killingworth   \n",
       "...                         ...           ...           ...   \n",
       "226220                      760            CT  East Hampton   \n",
       "226221                      760            CT  wethersfield   \n",
       "226222                      760            CT  wethersfield   \n",
       "226223                      760            CT  wethersfield   \n",
       "226224                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60              True         False             True   \n",
       "1                        66             False         False             True   \n",
       "2                         5             False         False            False   \n",
       "3                        15             False         False            False   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "226220                    0             False         False            False   \n",
       "226221                  100             False         False             True   \n",
       "226222                  100             False         False             True   \n",
       "226223                  100             False         False             True   \n",
       "226224                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "226220                       Gas  \n",
       "226221                  Electric  \n",
       "226222                  Electric  \n",
       "226223                  Electric  \n",
       "226224                  Electric  \n",
       "\n",
       "[217112 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/CT/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/CT-day/2018-jul-day-CT.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4c5caa1e348acf2719f577d8a4727442ca33b60d</td>\n",
       "      <td>2018-07-02 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>CT</td>\n",
       "      <td>Barkhamsted</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-07-20 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2018-07-27 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>755</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7e57d8be21393036f10df57e000165b31fed3576</td>\n",
       "      <td>2018-07-05 10:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>824</td>\n",
       "      <td>830</td>\n",
       "      <td>810</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Haven</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-07-06 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427059</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2018-07-21 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427060</th>\n",
       "      <td>6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb</td>\n",
       "      <td>2018-07-25 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>68</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427061</th>\n",
       "      <td>6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb</td>\n",
       "      <td>2018-07-05 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>68</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427062</th>\n",
       "      <td>6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb</td>\n",
       "      <td>2018-07-25 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>68</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427063</th>\n",
       "      <td>6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb</td>\n",
       "      <td>2018-07-25 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>68</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394293 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       4c5caa1e348acf2719f577d8a4727442ca33b60d  2018-07-02 11:40:00 UTC   \n",
       "2       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-07-20 17:10:00 UTC   \n",
       "3       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2018-07-27 16:20:00 UTC   \n",
       "4       7e57d8be21393036f10df57e000165b31fed3576  2018-07-05 10:15:00 UTC   \n",
       "5       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-07-06 14:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "427059  966ada58bdafef93eef95ff34ccb024ef8052f62  2018-07-21 19:10:00 UTC   \n",
       "427060  6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb  2018-07-25 19:20:00 UTC   \n",
       "427061  6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb  2018-07-05 17:35:00 UTC   \n",
       "427062  6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb  2018-07-25 19:35:00 UTC   \n",
       "427063  6c05c45d90fd4a9c36fbe9a37f7d0f7aacc2d6eb  2018-07-25 19:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               758                      755   \n",
       "2          cool          hold               713                      712   \n",
       "3          cool          auto               774                      770   \n",
       "4          cool          hold               824                      830   \n",
       "5          cool          hold               725                      762   \n",
       "...         ...           ...               ...                      ...   \n",
       "427059     cool          hold               746                      760   \n",
       "427060     cool          hold               753                      760   \n",
       "427061     cool          auto               760                      770   \n",
       "427062     cool          hold               757                      760   \n",
       "427063     cool          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           755            CT    Barkhamsted   \n",
       "2                           712            CT      Woodstock   \n",
       "3                           755            CT        Windsor   \n",
       "4                           810            CT     West Haven   \n",
       "5                           762            CT      Woodstock   \n",
       "...                         ...           ...            ...   \n",
       "427059                      760            CT   wethersfield   \n",
       "427060                      760            CT  West Hartford   \n",
       "427061                      760            CT  West Hartford   \n",
       "427062                      760            CT  West Hartford   \n",
       "427063                      760            CT  West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        20              True         False             True   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "427059                  100             False         False             True   \n",
       "427060                   68             False         False            False   \n",
       "427061                   68             False         False            False   \n",
       "427062                   68             False         False            False   \n",
       "427063                   68             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "427059                  Electric  \n",
       "427060                       Gas  \n",
       "427061                       Gas  \n",
       "427062                       Gas  \n",
       "427063                       Gas  \n",
       "\n",
       "[394293 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/CT/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/CT-day/2019-jul-day-CT.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7d20b90cb0a2faa613c7635d9bb1b6e86d22cc13</td>\n",
       "      <td>2019-07-06 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>764</td>\n",
       "      <td>713</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-07-31 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-07-07 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-07-11 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-07-21 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500663</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-07-21 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500664</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-07-21 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500665</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-07-04 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500666</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-07-14 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500667</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2019-07-04 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>469385 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7d20b90cb0a2faa613c7635d9bb1b6e86d22cc13  2019-07-06 18:00:00 UTC   \n",
       "1       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-07-31 17:30:00 UTC   \n",
       "2       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-07-07 15:35:00 UTC   \n",
       "3       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-07-11 13:25:00 UTC   \n",
       "4       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-07-21 13:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "500663  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-07-21 17:30:00 UTC   \n",
       "500664  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-07-21 18:30:00 UTC   \n",
       "500665  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-07-04 17:05:00 UTC   \n",
       "500666  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-07-14 19:55:00 UTC   \n",
       "500667  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2019-07-04 15:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               762                      764   \n",
       "1          cool          hold               720                      715   \n",
       "2          cool          hold               705                      705   \n",
       "3          cool          hold               705                      715   \n",
       "4          cool          hold               725                      725   \n",
       "...         ...           ...               ...                      ...   \n",
       "500663     cool          hold               760                      760   \n",
       "500664     cool          hold               763                      760   \n",
       "500665     cool          hold               760                      760   \n",
       "500666     cool          auto               764                      760   \n",
       "500667     cool          hold               757                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           713            CT         Monroe   \n",
       "1                           715            CT      Waterford   \n",
       "2                           705            CT      Waterford   \n",
       "3                           715            CT      Waterford   \n",
       "4                           725            CT      Waterford   \n",
       "...                         ...           ...            ...   \n",
       "500663                      760            CT  West Hartford   \n",
       "500664                      760            CT  West Hartford   \n",
       "500665                      760            CT  West Hartford   \n",
       "500666                      760            CT  West Hartford   \n",
       "500667                      760            CT  West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        25             False         False             True   \n",
       "2                        25             False         False             True   \n",
       "3                        25             False         False             True   \n",
       "4                        25             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "500663                   75              True         False            False   \n",
       "500664                   75              True         False            False   \n",
       "500665                   75              True         False            False   \n",
       "500666                   75              True         False            False   \n",
       "500667                   75              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "500663                       Gas  \n",
       "500664                       Gas  \n",
       "500665                       Gas  \n",
       "500666                       Gas  \n",
       "500667                       Gas  \n",
       "\n",
       "[469385 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/CT/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/CT-day/2020-jul-day-CT.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b3c0ff17ca5abe920bd4b20969fcf37806cd940d</td>\n",
       "      <td>2020-07-11 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ellington</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c147c2e1cbe076bc976db8e788fa285e351f07db</td>\n",
       "      <td>2020-07-08 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>749</td>\n",
       "      <td>744</td>\n",
       "      <td>732</td>\n",
       "      <td>CT</td>\n",
       "      <td>Durham</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>81b9f64b6f86312ce534fc4a17536a498b3d48a6</td>\n",
       "      <td>2020-07-01 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>727</td>\n",
       "      <td>713</td>\n",
       "      <td>CT</td>\n",
       "      <td>East Hartford</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>97b4e7f717297d6303e705e50cd9e5b83035da86</td>\n",
       "      <td>2020-07-27 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1d643e73a2c013cb3b79b099d91d3e32d25ebf7f</td>\n",
       "      <td>2020-07-25 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Stonington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527109</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-07-16 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527110</th>\n",
       "      <td>467bba8253f6e67dc1327f0ac0d681f5c7115c83</td>\n",
       "      <td>2020-07-16 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527111</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-07-11 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527112</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-07-16 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527113</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-07-26 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>508280 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b3c0ff17ca5abe920bd4b20969fcf37806cd940d  2020-07-11 18:55:00 UTC   \n",
       "1       c147c2e1cbe076bc976db8e788fa285e351f07db  2020-07-08 15:25:00 UTC   \n",
       "2       81b9f64b6f86312ce534fc4a17536a498b3d48a6  2020-07-01 17:50:00 UTC   \n",
       "3       97b4e7f717297d6303e705e50cd9e5b83035da86  2020-07-27 16:10:00 UTC   \n",
       "5       1d643e73a2c013cb3b79b099d91d3e32d25ebf7f  2020-07-25 17:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "527109  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-07-16 17:10:00 UTC   \n",
       "527110  467bba8253f6e67dc1327f0ac0d681f5c7115c83  2020-07-16 19:45:00 UTC   \n",
       "527111  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-07-11 19:45:00 UTC   \n",
       "527112  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-07-16 15:45:00 UTC   \n",
       "527113  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-07-26 19:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               735                      715   \n",
       "1          cool          auto               749                      744   \n",
       "2          cool          auto               737                      727   \n",
       "3          cool          auto               741                      760   \n",
       "5          cool          hold               776                      731   \n",
       "...         ...           ...               ...                      ...   \n",
       "527109     cool          auto               715                      760   \n",
       "527110     cool          auto               748                      760   \n",
       "527111     cool          auto               754                      760   \n",
       "527112     cool          auto               706                      760   \n",
       "527113     cool          auto               747                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           715            CT      Ellington   \n",
       "1                           732            CT         Durham   \n",
       "2                           713            CT  East Hartford   \n",
       "3                           731            CT         Monroe   \n",
       "5                           731            CT     Stonington   \n",
       "...                         ...           ...            ...   \n",
       "527109                      760            CT  West Hartford   \n",
       "527110                      760            CT  West Hartford   \n",
       "527111                      760            CT  West Hartford   \n",
       "527112                      760            CT  West Hartford   \n",
       "527113                      760            CT  West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        50             False         False            False   \n",
       "3                        39              True         False            False   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "527109                   56             False         False            False   \n",
       "527110                   56             False         False            False   \n",
       "527111                   56             False         False            False   \n",
       "527112                   56             False         False            False   \n",
       "527113                   56             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "527109                       Gas  \n",
       "527110                       Gas  \n",
       "527111                       Gas  \n",
       "527112                       Gas  \n",
       "527113                       Gas  \n",
       "\n",
       "[508280 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/CT/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/CT-day/2021-jul-day-CT.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>215c2dfd21213f343402901d6a5bee6dae1c5718</td>\n",
       "      <td>2021-07-17 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>777</td>\n",
       "      <td>770</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e9ecbea95473c9c90df38e5c762652f6d9ca5c4c</td>\n",
       "      <td>2021-07-13 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Easton</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>215c2dfd21213f343402901d6a5bee6dae1c5718</td>\n",
       "      <td>2021-07-17 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>783</td>\n",
       "      <td>770</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71b7a96cb3b5e2af708eb1e817f3e24a0023cb51</td>\n",
       "      <td>2021-07-21 11:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Farmington</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>215c2dfd21213f343402901d6a5bee6dae1c5718</td>\n",
       "      <td>2021-07-19 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>770</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323907</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-07-02 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323908</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-07-07 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323909</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-07-02 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323910</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-07-16 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323911</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2021-07-02 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>308205 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       215c2dfd21213f343402901d6a5bee6dae1c5718  2021-07-17 18:40:00 UTC   \n",
       "1       e9ecbea95473c9c90df38e5c762652f6d9ca5c4c  2021-07-13 17:00:00 UTC   \n",
       "2       215c2dfd21213f343402901d6a5bee6dae1c5718  2021-07-17 19:20:00 UTC   \n",
       "3       71b7a96cb3b5e2af708eb1e817f3e24a0023cb51  2021-07-21 11:30:00 UTC   \n",
       "4       215c2dfd21213f343402901d6a5bee6dae1c5718  2021-07-19 13:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "323907  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-07-02 17:20:00 UTC   \n",
       "323908  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-07-07 19:50:00 UTC   \n",
       "323909  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-07-02 18:20:00 UTC   \n",
       "323910  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-07-16 18:10:00 UTC   \n",
       "323911  966ada58bdafef93eef95ff34ccb024ef8052f62  2021-07-02 19:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               777                      770   \n",
       "1          cool          hold               710                      705   \n",
       "2          auto          hold               783                      770   \n",
       "3          cool          hold               715                      731   \n",
       "4          auto          hold               752                      770   \n",
       "...         ...           ...               ...                      ...   \n",
       "323907     cool          hold               742                      760   \n",
       "323908     cool          hold               745                      760   \n",
       "323909     cool          hold               740                      760   \n",
       "323910     cool          hold               762                      760   \n",
       "323911     cool          hold               742                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           675            CT        Orange   \n",
       "1                           705            CT        Easton   \n",
       "2                           675            CT        Orange   \n",
       "3                           731            CT    Farmington   \n",
       "4                           675            CT        Orange   \n",
       "...                         ...           ...           ...   \n",
       "323907                      760            CT  wethersfield   \n",
       "323908                      760            CT  wethersfield   \n",
       "323909                      760            CT  wethersfield   \n",
       "323910                      760            CT  wethersfield   \n",
       "323911                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                        60             False         False             True   \n",
       "2                         0              True         False            False   \n",
       "3                        20              True         False            False   \n",
       "4                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "323907                  100             False         False             True   \n",
       "323908                  100             False         False             True   \n",
       "323909                  100             False         False             True   \n",
       "323910                  100             False         False             True   \n",
       "323911                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "323907                  Electric  \n",
       "323908                  Electric  \n",
       "323909                  Electric  \n",
       "323910                  Electric  \n",
       "323911                  Electric  \n",
       "\n",
       "[308205 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/CT/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>007f9b37f1ac5cfe9806f9a1a47809855e7e31b8</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Windsor</td>\n",
       "      <td>721.520971</td>\n",
       "      <td>727.560706</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>721.624161</td>\n",
       "      <td>763.489933</td>\n",
       "      <td>642.751678</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>720.050556</td>\n",
       "      <td>740.758342</td>\n",
       "      <td>651.882710</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>728.125000</td>\n",
       "      <td>719.375000</td>\n",
       "      <td>713.875000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>011dae903f5512427598b14bbb3f02baa8813745</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>765.666667</td>\n",
       "      <td>732.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350</th>\n",
       "      <td>fc92f3d7ab046b1e0482835f5cf10e18372dc434</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>700.558824</td>\n",
       "      <td>690.397059</td>\n",
       "      <td>689.794118</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>351</th>\n",
       "      <td>fdce2932511e0bf91c9516b4f56bba7ca424f147</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>745.000000</td>\n",
       "      <td>716.666667</td>\n",
       "      <td>663.333333</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>fdda1e11626a48045c202219e652ae8b195c85b1</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sandy Hook</td>\n",
       "      <td>722.970588</td>\n",
       "      <td>721.176471</td>\n",
       "      <td>721.176471</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>658.626189</td>\n",
       "      <td>655.972733</td>\n",
       "      <td>655.955295</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>354</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>728.613017</td>\n",
       "      <td>733.839050</td>\n",
       "      <td>733.839050</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2640 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    007f9b37f1ac5cfe9806f9a1a47809855e7e31b8   jul  2017     cool   \n",
       "1    0094e2c467552f4b15765889040931605fb22890   jul  2017     auto   \n",
       "2    0094e2c467552f4b15765889040931605fb22890   jul  2017     cool   \n",
       "3    0094e2c467552f4b15765889040931605fb22890   jul  2017     cool   \n",
       "4    011dae903f5512427598b14bbb3f02baa8813745   jul  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "350  fc92f3d7ab046b1e0482835f5cf10e18372dc434   jul  2021     cool   \n",
       "351  fdce2932511e0bf91c9516b4f56bba7ca424f147   jul  2021     auto   \n",
       "352  fdda1e11626a48045c202219e652ae8b195c85b1   jul  2021     cool   \n",
       "353  fef94e274991646e0f28d91d1aaea3cd709d8825   jul  2021     cool   \n",
       "354  ff498c11fdbb5d698084e0f03a29b69e76ee8566   jul  2021     cool   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold  South Windsor            721.520971   \n",
       "1            auto        Norwalk            721.624161   \n",
       "2            auto        Norwalk            720.050556   \n",
       "3            hold        Norwalk            728.125000   \n",
       "4            auto        Shelton            765.666667   \n",
       "..            ...            ...                   ...   \n",
       "350          hold      Fairfield            700.558824   \n",
       "351          hold        Shelton            745.000000   \n",
       "352          hold     Sandy Hook            722.970588   \n",
       "353          hold     New Canaan            658.626189   \n",
       "354          hold    North Haven            728.613017   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     727.560706                   660.000000   \n",
       "1                     763.489933                   642.751678   \n",
       "2                     740.758342                   651.882710   \n",
       "3                     719.375000                   713.875000   \n",
       "4                     732.000000                   650.000000   \n",
       "..                           ...                          ...   \n",
       "350                   690.397059                   689.794118   \n",
       "351                   716.666667                   663.333333   \n",
       "352                   721.176471                   721.176471   \n",
       "353                   655.972733                   655.955295   \n",
       "354                   733.839050                   733.839050   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   27.0             False         False            False  \n",
       "2                   27.0             False         False            False  \n",
       "3                   27.0             False         False            False  \n",
       "4                    5.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "350                 40.0              True         False             True  \n",
       "351                  5.0             False         False            False  \n",
       "352                 20.0             False         False            False  \n",
       "353                  0.0             False         False            False  \n",
       "354                 70.0             False         False            False  \n",
       "\n",
       "[2640 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/jul/\" + file)\n",
    "    CT_jul = pd.concat([CT_jul, df])\n",
    "    \n",
    "CT_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_jul.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/CT-day/2017-aug-day-CT.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2017-08-27 10:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>724</td>\n",
       "      <td>674</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b4061b7ca02df9d17e37c1919645ce998c0b568d</td>\n",
       "      <td>2017-08-27 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>734</td>\n",
       "      <td>684</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b4061b7ca02df9d17e37c1919645ce998c0b568d</td>\n",
       "      <td>2017-08-29 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>734</td>\n",
       "      <td>684</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2017-08-28 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>734</td>\n",
       "      <td>674</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>564c6aec11077a013c08daef6f741cea54a99227</td>\n",
       "      <td>2017-08-14 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>704</td>\n",
       "      <td>746</td>\n",
       "      <td>CT</td>\n",
       "      <td>Southbury</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215915</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-08-02 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215916</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-08-23 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215917</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-08-09 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215918</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-08-09 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>215919</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2017-08-09 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208324 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2017-08-27 10:30:00 UTC   \n",
       "1       b4061b7ca02df9d17e37c1919645ce998c0b568d  2017-08-27 17:00:00 UTC   \n",
       "2       b4061b7ca02df9d17e37c1919645ce998c0b568d  2017-08-29 15:25:00 UTC   \n",
       "3       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2017-08-28 12:30:00 UTC   \n",
       "4       564c6aec11077a013c08daef6f741cea54a99227  2017-08-14 11:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "215915  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-08-02 19:35:00 UTC   \n",
       "215916  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-08-23 19:45:00 UTC   \n",
       "215917  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-08-09 17:00:00 UTC   \n",
       "215918  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-08-09 16:10:00 UTC   \n",
       "215919  966ada58bdafef93eef95ff34ccb024ef8052f62  2017-08-09 13:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               713                      724   \n",
       "1          auto          hold               712                      734   \n",
       "2          auto          hold               705                      734   \n",
       "3          auto          hold               718                      734   \n",
       "4          cool          auto               680                      704   \n",
       "...         ...           ...               ...                      ...   \n",
       "215915     cool          hold               747                      760   \n",
       "215916     cool          hold               744                      760   \n",
       "215917     cool          hold               745                      760   \n",
       "215918     cool          hold               739                      760   \n",
       "215919     cool          hold               721                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           674            CT      Woodbury   \n",
       "1                           684            CT      Woodbury   \n",
       "2                           684            CT      Woodbury   \n",
       "3                           674            CT      Woodbury   \n",
       "4                           746            CT     Southbury   \n",
       "...                         ...           ...           ...   \n",
       "215915                      760            CT  wethersfield   \n",
       "215916                      760            CT  wethersfield   \n",
       "215917                      760            CT  wethersfield   \n",
       "215918                      760            CT  wethersfield   \n",
       "215919                      760            CT  wethersfield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        40             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "215915                  100             False         False             True   \n",
       "215916                  100             False         False             True   \n",
       "215917                  100             False         False             True   \n",
       "215918                  100             False         False             True   \n",
       "215919                  100             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "215915                  Electric  \n",
       "215916                  Electric  \n",
       "215917                  Electric  \n",
       "215918                  Electric  \n",
       "215919                  Electric  \n",
       "\n",
       "[208324 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/CT/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/CT-day/2018-aug-day-CT.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-08-28 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2018-08-15 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>755</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7e57d8be21393036f10df57e000165b31fed3576</td>\n",
       "      <td>2018-08-26 07:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>828</td>\n",
       "      <td>840</td>\n",
       "      <td>810</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Haven</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-08-28 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-08-21 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417989</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2018-08-16 10:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417990</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2018-08-17 10:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417991</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2018-08-17 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417992</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2018-08-20 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417993</th>\n",
       "      <td>7d4a998520c8f7836cbde9e5d1321bbe2f945076</td>\n",
       "      <td>2018-08-16 10:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>396176 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-08-28 18:30:00 UTC   \n",
       "1       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2018-08-15 12:40:00 UTC   \n",
       "2       7e57d8be21393036f10df57e000165b31fed3576  2018-08-26 07:10:00 UTC   \n",
       "3       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-08-28 18:45:00 UTC   \n",
       "4       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-08-21 12:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "417989  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2018-08-16 10:30:00 UTC   \n",
       "417990  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2018-08-17 10:00:00 UTC   \n",
       "417991  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2018-08-17 18:55:00 UTC   \n",
       "417992  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2018-08-20 12:15:00 UTC   \n",
       "417993  7d4a998520c8f7836cbde9e5d1321bbe2f945076  2018-08-16 10:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               726                      725   \n",
       "1          auto          hold               749                      755   \n",
       "2          cool          hold               828                      840   \n",
       "3          cool          hold               728                      752   \n",
       "4          auto          auto               711                      725   \n",
       "...         ...           ...               ...                      ...   \n",
       "417989     cool          hold               761                      760   \n",
       "417990     cool          auto               759                      760   \n",
       "417991     cool          auto               763                      760   \n",
       "417992     cool          auto               750                      760   \n",
       "417993     cool          hold               762                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           675            CT    Broad Brook   \n",
       "1                           685            CT       Woodbury   \n",
       "2                           810            CT     West Haven   \n",
       "3                           752            CT      Woodstock   \n",
       "4                           675            CT    Broad Brook   \n",
       "...                         ...           ...            ...   \n",
       "417989                      760            CT  West Hartford   \n",
       "417990                      760            CT  West Hartford   \n",
       "417991                      760            CT  West Hartford   \n",
       "417992                      760            CT  West Hartford   \n",
       "417993                      760            CT  West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60             False         False            False   \n",
       "1                        30             False         False            False   \n",
       "2                        20              True         False             True   \n",
       "3                        10             False         False            False   \n",
       "4                        60             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "417989                   75              True         False            False   \n",
       "417990                   75              True         False            False   \n",
       "417991                   75              True         False            False   \n",
       "417992                   75              True         False            False   \n",
       "417993                   75              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "417989                       Gas  \n",
       "417990                       Gas  \n",
       "417991                       Gas  \n",
       "417992                       Gas  \n",
       "417993                       Gas  \n",
       "\n",
       "[396176 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/CT/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/CT-day/2019-aug-day-CT.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-08-25 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0382f14f425de2d9f9645def17ee3cfeada55f78</td>\n",
       "      <td>2019-08-30 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Waterford</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b9eb493652b04bdbd9a25c27c17a34f3e00646cd</td>\n",
       "      <td>2019-08-26 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>722</td>\n",
       "      <td>737</td>\n",
       "      <td>687</td>\n",
       "      <td>CT</td>\n",
       "      <td>Orange</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d4db52a911b76755478ebb3cd77a5cfea9e3007b</td>\n",
       "      <td>2019-08-19 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>CT</td>\n",
       "      <td>Bloomfield</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>249b0f3298602e0d392735958afe0b536033998b</td>\n",
       "      <td>2019-08-11 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>780</td>\n",
       "      <td>778</td>\n",
       "      <td>CT</td>\n",
       "      <td>Cromwell</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455061</th>\n",
       "      <td>889dcbf43cea92e9262ed9971030d156a71de5f5</td>\n",
       "      <td>2019-08-10 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>CT</td>\n",
       "      <td>Mystic</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455062</th>\n",
       "      <td>6530ef0335c4d41524283ecfa5d1fc0e1fd25f55</td>\n",
       "      <td>2019-08-05 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>805</td>\n",
       "      <td>800</td>\n",
       "      <td>700</td>\n",
       "      <td>CT</td>\n",
       "      <td>Killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455063</th>\n",
       "      <td>b92711525879af78592f41c536de1ace727e4dc2</td>\n",
       "      <td>2019-08-23 10:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ellington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455064</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2019-08-04 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455065</th>\n",
       "      <td>6530ef0335c4d41524283ecfa5d1fc0e1fd25f55</td>\n",
       "      <td>2019-08-05 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>700</td>\n",
       "      <td>CT</td>\n",
       "      <td>Killingworth</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>423917 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-08-25 13:25:00 UTC   \n",
       "1       0382f14f425de2d9f9645def17ee3cfeada55f78  2019-08-30 18:10:00 UTC   \n",
       "2       b9eb493652b04bdbd9a25c27c17a34f3e00646cd  2019-08-26 19:40:00 UTC   \n",
       "3       d4db52a911b76755478ebb3cd77a5cfea9e3007b  2019-08-19 12:45:00 UTC   \n",
       "4       249b0f3298602e0d392735958afe0b536033998b  2019-08-11 15:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "455061  889dcbf43cea92e9262ed9971030d156a71de5f5  2019-08-10 14:35:00 UTC   \n",
       "455062  6530ef0335c4d41524283ecfa5d1fc0e1fd25f55  2019-08-05 12:40:00 UTC   \n",
       "455063  b92711525879af78592f41c536de1ace727e4dc2  2019-08-23 10:00:00 UTC   \n",
       "455064  528482e17661477da3fa31d0b3d158c6fb72e9aa  2019-08-04 13:55:00 UTC   \n",
       "455065  6530ef0335c4d41524283ecfa5d1fc0e1fd25f55  2019-08-05 12:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               724                      715   \n",
       "1          cool          hold               702                      685   \n",
       "2          auto          auto               722                      737   \n",
       "3          cool          hold               710                      712   \n",
       "4          cool          hold               756                      780   \n",
       "...         ...           ...               ...                      ...   \n",
       "455061     cool          hold               711                      700   \n",
       "455062     auto          hold               805                      800   \n",
       "455063     cool          auto               703                      700   \n",
       "455064     cool          hold               703                      700   \n",
       "455065     auto          hold               800                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           715            CT     Waterford   \n",
       "1                           685            CT     Waterford   \n",
       "2                           687            CT        Orange   \n",
       "3                           712            CT    Bloomfield   \n",
       "4                           778            CT      Cromwell   \n",
       "...                         ...           ...           ...   \n",
       "455061                      700            CT        Mystic   \n",
       "455062                      700            CT  Killingworth   \n",
       "455063                      700            CT     Ellington   \n",
       "455064                      700            CT     Woodstock   \n",
       "455065                      700            CT  Killingworth   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False             True   \n",
       "1                        25             False         False             True   \n",
       "2                        55             False         False            False   \n",
       "3                        30              True         False             True   \n",
       "4                        35              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "455061                   49             False         False            False   \n",
       "455062                   15             False         False            False   \n",
       "455063                   15             False         False            False   \n",
       "455064                   10             False         False            False   \n",
       "455065                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "455061                       Gas  \n",
       "455062                       Gas  \n",
       "455063                       Gas  \n",
       "455064                       Gas  \n",
       "455065                       Gas  \n",
       "\n",
       "[423917 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/CT/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/CT-day/2020-aug-day-CT.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2020-08-10 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1c755336958b8023df4fceca6edfabb466deaef5</td>\n",
       "      <td>2020-08-01 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>628</td>\n",
       "      <td>632</td>\n",
       "      <td>632</td>\n",
       "      <td>CT</td>\n",
       "      <td>Mansfield Center</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2020-08-27 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cdc4e0fa4b56e51585b5a93bae44b51150ca4860</td>\n",
       "      <td>2020-08-15 10:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor Locks</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0090bb6fc40c84b7f42c125689e22350e6128e3e</td>\n",
       "      <td>2020-08-28 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>740</td>\n",
       "      <td>731</td>\n",
       "      <td>CT</td>\n",
       "      <td>Monroe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462249</th>\n",
       "      <td>966ada58bdafef93eef95ff34ccb024ef8052f62</td>\n",
       "      <td>2020-08-22 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>wethersfield</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462250</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-08-25 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462251</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-08-25 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462252</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-08-25 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>462253</th>\n",
       "      <td>af6867edfa2849e1718a669e29fb2ae5f04852fd</td>\n",
       "      <td>2020-08-25 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>West Hartford</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>447994 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2020-08-10 14:55:00 UTC   \n",
       "1       1c755336958b8023df4fceca6edfabb466deaef5  2020-08-01 15:10:00 UTC   \n",
       "2       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2020-08-27 13:45:00 UTC   \n",
       "3       cdc4e0fa4b56e51585b5a93bae44b51150ca4860  2020-08-15 10:25:00 UTC   \n",
       "4       0090bb6fc40c84b7f42c125689e22350e6128e3e  2020-08-28 19:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "462249  966ada58bdafef93eef95ff34ccb024ef8052f62  2020-08-22 16:45:00 UTC   \n",
       "462250  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-08-25 17:15:00 UTC   \n",
       "462251  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-08-25 17:35:00 UTC   \n",
       "462252  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-08-25 17:25:00 UTC   \n",
       "462253  af6867edfa2849e1718a669e29fb2ae5f04852fd  2020-08-25 18:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               689                      685   \n",
       "1          cool          hold               628                      632   \n",
       "2          auto          hold               687                      685   \n",
       "3          auto          hold               689                      685   \n",
       "4          cool          hold               746                      740   \n",
       "...         ...           ...               ...                      ...   \n",
       "462249     cool          hold               737                      760   \n",
       "462250     cool          auto               734                      760   \n",
       "462251     cool          auto               738                      760   \n",
       "462252     cool          auto               736                      760   \n",
       "462253     cool          auto               742                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                           635            CT     Windsor Locks   \n",
       "1                           632            CT  Mansfield Center   \n",
       "2                           635            CT     Windsor Locks   \n",
       "3                           635            CT     Windsor Locks   \n",
       "4                           731            CT            Monroe   \n",
       "...                         ...           ...               ...   \n",
       "462249                      760            CT      wethersfield   \n",
       "462250                      760            CT     West Hartford   \n",
       "462251                      760            CT     West Hartford   \n",
       "462252                      760            CT     West Hartford   \n",
       "462253                      760            CT     West Hartford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                         0              True         False             True   \n",
       "2                        30             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "462249                  100             False         False             True   \n",
       "462250                   56             False         False            False   \n",
       "462251                   56             False         False            False   \n",
       "462252                   56             False         False            False   \n",
       "462253                   56             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "462249                  Electric  \n",
       "462250                       Gas  \n",
       "462251                       Gas  \n",
       "462252                       Gas  \n",
       "462253                       Gas  \n",
       "\n",
       "[447994 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/CT/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>007f9b37f1ac5cfe9806f9a1a47809855e7e31b8</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Windsor</td>\n",
       "      <td>720.473373</td>\n",
       "      <td>720.727811</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>720.490196</td>\n",
       "      <td>718.400000</td>\n",
       "      <td>650.200000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>721.850000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>011dae903f5512427598b14bbb3f02baa8813745</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>784.868922</td>\n",
       "      <td>780.002114</td>\n",
       "      <td>620.002114</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01440e0a70900810dceaace98a7965a75bc818dc</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>727.825439</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>fe0e87faf88e04d7b4bbaaa926867d237a8c0b80</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westport</td>\n",
       "      <td>742.250000</td>\n",
       "      <td>725.000000</td>\n",
       "      <td>725.000000</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>614</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>676.085106</td>\n",
       "      <td>673.591380</td>\n",
       "      <td>619.991271</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>fefe9a1e7cdca41851d594cf74b2d28a98aca1de</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Plantsville</td>\n",
       "      <td>720.607143</td>\n",
       "      <td>705.511905</td>\n",
       "      <td>704.988095</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>749.138889</td>\n",
       "      <td>751.601852</td>\n",
       "      <td>751.601852</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>750.069655</td>\n",
       "      <td>761.233473</td>\n",
       "      <td>761.233473</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2136 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    007f9b37f1ac5cfe9806f9a1a47809855e7e31b8   aug  2017     cool   \n",
       "1    0094e2c467552f4b15765889040931605fb22890   aug  2017     cool   \n",
       "2    0094e2c467552f4b15765889040931605fb22890   aug  2017     cool   \n",
       "3    011dae903f5512427598b14bbb3f02baa8813745   aug  2017     auto   \n",
       "4    01440e0a70900810dceaace98a7965a75bc818dc   aug  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "613  fe0e87faf88e04d7b4bbaaa926867d237a8c0b80   aug  2020     cool   \n",
       "614  fef94e274991646e0f28d91d1aaea3cd709d8825   aug  2020     cool   \n",
       "615  fefe9a1e7cdca41851d594cf74b2d28a98aca1de   aug  2020     cool   \n",
       "616  ff498c11fdbb5d698084e0f03a29b69e76ee8566   aug  2020     cool   \n",
       "617  ff498c11fdbb5d698084e0f03a29b69e76ee8566   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold  South Windsor            720.473373   \n",
       "1            auto        Norwalk            720.490196   \n",
       "2            hold        Norwalk            721.850000   \n",
       "3            auto        Shelton            784.868922   \n",
       "4            auto       Guilford            727.825439   \n",
       "..            ...            ...                   ...   \n",
       "613          hold       Westport            742.250000   \n",
       "614          hold     New Canaan            676.085106   \n",
       "615          hold    Plantsville            720.607143   \n",
       "616          auto    North Haven            749.138889   \n",
       "617          hold    North Haven            750.069655   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     720.727811                   660.000000   \n",
       "1                     718.400000                   650.200000   \n",
       "2                     710.000000                   710.000000   \n",
       "3                     780.002114                   620.002114   \n",
       "4                     740.000000                   730.000000   \n",
       "..                           ...                          ...   \n",
       "613                   725.000000                   725.000000   \n",
       "614                   673.591380                   619.991271   \n",
       "615                   705.511905                   704.988095   \n",
       "616                   751.601852                   751.601852   \n",
       "617                   761.233473                   761.233473   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   27.0             False         False            False  \n",
       "2                   27.0             False         False            False  \n",
       "3                    5.0             False         False            False  \n",
       "4                    0.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "613                 40.0             False         False            False  \n",
       "614                  0.0             False         False            False  \n",
       "615                 10.0             False         False            False  \n",
       "616                 70.0             False         False            False  \n",
       "617                 70.0             False         False            False  \n",
       "\n",
       "[2136 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/aug/\" + file)\n",
    "    CT_aug = pd.concat([CT_aug, df])\n",
    "    \n",
    "CT_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_aug.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/CT-day/2017-dec-day-CT.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2017-12-27 10:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1e6ca6fd677aff1dd39019c596be93cefd8a2ac7</td>\n",
       "      <td>2017-12-14 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ansonia</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2017-12-04 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cd8d4e49e9275cb2b70333d2e4f6feab86b3c681</td>\n",
       "      <td>2017-12-19 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>674</td>\n",
       "      <td>820</td>\n",
       "      <td>624</td>\n",
       "      <td>CT</td>\n",
       "      <td>killingworth</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2017-12-14 10:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355910</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-12-31 11:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355911</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-12-31 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355912</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-12-30 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355913</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-12-31 12:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355914</th>\n",
       "      <td>628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7</td>\n",
       "      <td>2017-12-30 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>326692 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2017-12-27 10:35:00 UTC   \n",
       "1       1e6ca6fd677aff1dd39019c596be93cefd8a2ac7  2017-12-14 16:05:00 UTC   \n",
       "3       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2017-12-04 13:25:00 UTC   \n",
       "4       cd8d4e49e9275cb2b70333d2e4f6feab86b3c681  2017-12-19 13:30:00 UTC   \n",
       "6       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2017-12-14 10:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "355910  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-12-31 11:30:00 UTC   \n",
       "355911  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-12-31 12:40:00 UTC   \n",
       "355912  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-12-30 11:35:00 UTC   \n",
       "355913  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-12-31 12:15:00 UTC   \n",
       "355914  628c5a78fbbbace955cfd5e8c0e384c28e9c0fd7  2017-12-30 14:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               699                      705   \n",
       "1          heat          hold               660                      665   \n",
       "3          auto          hold               704                      755   \n",
       "4          heat          auto               674                      820   \n",
       "6          auto          hold               700                      755   \n",
       "...         ...           ...               ...                      ...   \n",
       "355910     heat          auto               753                      760   \n",
       "355911     heat          auto               759                      760   \n",
       "355912     heat          auto               759                      760   \n",
       "355913     heat          auto               760                      760   \n",
       "355914     heat          auto               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           705            CT      Woodbury   \n",
       "1                           665            CT       Ansonia   \n",
       "3                           705            CT      Woodbury   \n",
       "4                           624            CT  killingworth   \n",
       "6                           705            CT      Woodbury   \n",
       "...                         ...           ...           ...   \n",
       "355910                      760            CT       Norwalk   \n",
       "355911                      760            CT       Norwalk   \n",
       "355912                      760            CT       Norwalk   \n",
       "355913                      760            CT       Norwalk   \n",
       "355914                      760            CT       Norwalk   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        25             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        20             False         False            False   \n",
       "6                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "355910                    5             False         False            False   \n",
       "355911                    5             False         False            False   \n",
       "355912                    5             False         False            False   \n",
       "355913                    5             False         False            False   \n",
       "355914                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "355910                       Gas  \n",
       "355911                       Gas  \n",
       "355912                       Gas  \n",
       "355913                       Gas  \n",
       "355914                       Gas  \n",
       "\n",
       "[326692 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/CT/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/CT-day/2018-dec-day-CT.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-12-31 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>672</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-12-26 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>528482e17661477da3fa31d0b3d158c6fb72e9aa</td>\n",
       "      <td>2018-12-08 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>662</td>\n",
       "      <td>662</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodstock</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e9870d26f289a344a43ab52d729cb869f3baaa62</td>\n",
       "      <td>2018-12-21 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>685</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>CT</td>\n",
       "      <td>Broad Brook</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>476a76536d2ba66f48e71fa98aeb0ede4d434c87</td>\n",
       "      <td>2018-12-27 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>775</td>\n",
       "      <td>705</td>\n",
       "      <td>CT</td>\n",
       "      <td>Woodbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486076</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2018-12-27 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486077</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2018-12-30 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486078</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2018-12-30 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486079</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2018-12-27 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>486080</th>\n",
       "      <td>a6a33cee140a4ee6cc5c9b875fe15765800fea90</td>\n",
       "      <td>2018-12-30 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>431554 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-12-31 14:45:00 UTC   \n",
       "1       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-12-26 16:50:00 UTC   \n",
       "2       528482e17661477da3fa31d0b3d158c6fb72e9aa  2018-12-08 16:35:00 UTC   \n",
       "3       e9870d26f289a344a43ab52d729cb869f3baaa62  2018-12-21 17:05:00 UTC   \n",
       "4       476a76536d2ba66f48e71fa98aeb0ede4d434c87  2018-12-27 12:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "486076  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2018-12-27 18:50:00 UTC   \n",
       "486077  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2018-12-30 17:40:00 UTC   \n",
       "486078  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2018-12-30 19:10:00 UTC   \n",
       "486079  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2018-12-27 19:50:00 UTC   \n",
       "486080  a6a33cee140a4ee6cc5c9b875fe15765800fea90  2018-12-30 19:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               672                      725   \n",
       "1          auto          auto               675                      725   \n",
       "2          heat          hold               657                      662   \n",
       "3          auto          auto               685                      725   \n",
       "4          auto          hold               700                      775   \n",
       "...         ...           ...               ...                      ...   \n",
       "486076     auto          hold               749                      820   \n",
       "486077     auto          hold               748                      820   \n",
       "486078     auto          hold               750                      820   \n",
       "486079     auto          hold               762                      820   \n",
       "486080     auto          hold               749                      820   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           675            CT  Broad Brook   \n",
       "1                           675            CT  Broad Brook   \n",
       "2                           662            CT    Woodstock   \n",
       "3                           675            CT  Broad Brook   \n",
       "4                           705            CT     Woodbury   \n",
       "...                         ...           ...          ...   \n",
       "486076                      760            CT   Ridgefield   \n",
       "486077                      760            CT   Ridgefield   \n",
       "486078                      760            CT   Ridgefield   \n",
       "486079                      760            CT   Ridgefield   \n",
       "486080                      760            CT   Ridgefield   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60             False         False            False   \n",
       "1                        60             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "3                        60             False         False            False   \n",
       "4                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "486076                   50             False         False            False   \n",
       "486077                   50             False         False            False   \n",
       "486078                   50             False         False            False   \n",
       "486079                   50             False         False            False   \n",
       "486080                   50             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "486076                       Gas  \n",
       "486077                       Gas  \n",
       "486078                       Gas  \n",
       "486079                       Gas  \n",
       "486080                       Gas  \n",
       "\n",
       "[431554 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/CT/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/CT-day/2019-dec-day-CT.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4db52a911b76755478ebb3cd77a5cfea9e3007b</td>\n",
       "      <td>2019-12-12 10:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>CT</td>\n",
       "      <td>Bloomfield</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>edd88b332aea2fee543048ec1b27d65c159e4bab</td>\n",
       "      <td>2019-12-12 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>722</td>\n",
       "      <td>686</td>\n",
       "      <td>CT</td>\n",
       "      <td>Newtown</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d4db52a911b76755478ebb3cd77a5cfea9e3007b</td>\n",
       "      <td>2019-12-11 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>CT</td>\n",
       "      <td>Bloomfield</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>214f28562b2c6edb923553c2da33e34308417be2</td>\n",
       "      <td>2019-12-20 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>766</td>\n",
       "      <td>702</td>\n",
       "      <td>CT</td>\n",
       "      <td>Colchester</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>35b533b57d9a9b1f34214bef2b0e4373ebce9450</td>\n",
       "      <td>2019-12-13 11:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>627</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>CT</td>\n",
       "      <td>Windsor</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535491</th>\n",
       "      <td>2cc84ecbecd2da4c226c1569268fb8196a51343a</td>\n",
       "      <td>2019-12-19 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535492</th>\n",
       "      <td>2cc84ecbecd2da4c226c1569268fb8196a51343a</td>\n",
       "      <td>2019-12-19 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535493</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2019-12-23 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535494</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2019-12-23 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>535495</th>\n",
       "      <td>d844acbe1727e39f17d98fc6505c5afb592cb458</td>\n",
       "      <td>2019-12-23 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Torrington</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>487965 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d4db52a911b76755478ebb3cd77a5cfea9e3007b  2019-12-12 10:50:00 UTC   \n",
       "1       edd88b332aea2fee543048ec1b27d65c159e4bab  2019-12-12 14:55:00 UTC   \n",
       "4       d4db52a911b76755478ebb3cd77a5cfea9e3007b  2019-12-11 18:55:00 UTC   \n",
       "5       214f28562b2c6edb923553c2da33e34308417be2  2019-12-20 15:30:00 UTC   \n",
       "7       35b533b57d9a9b1f34214bef2b0e4373ebce9450  2019-12-13 11:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "535491  2cc84ecbecd2da4c226c1569268fb8196a51343a  2019-12-19 18:20:00 UTC   \n",
       "535492  2cc84ecbecd2da4c226c1569268fb8196a51343a  2019-12-19 16:00:00 UTC   \n",
       "535493  d844acbe1727e39f17d98fc6505c5afb592cb458  2019-12-23 13:55:00 UTC   \n",
       "535494  d844acbe1727e39f17d98fc6505c5afb592cb458  2019-12-23 14:05:00 UTC   \n",
       "535495  d844acbe1727e39f17d98fc6505c5afb592cb458  2019-12-23 14:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               673                      682   \n",
       "1          heat          hold               703                      722   \n",
       "4          heat          hold               681                      682   \n",
       "5          auto          hold               697                      766   \n",
       "7          heat          hold               627                      725   \n",
       "...         ...           ...               ...                      ...   \n",
       "535491     heat          hold               757                      760   \n",
       "535492     heat          hold               761                      760   \n",
       "535493     heat          hold               695                      760   \n",
       "535494     heat          hold               694                      760   \n",
       "535495     heat          hold               690                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           682            CT  Bloomfield   \n",
       "1                           686            CT     Newtown   \n",
       "4                           682            CT  Bloomfield   \n",
       "5                           702            CT  Colchester   \n",
       "7                           725            CT     Windsor   \n",
       "...                         ...           ...         ...   \n",
       "535491                      760            CT     Norwalk   \n",
       "535492                      760            CT     Norwalk   \n",
       "535493                      760            CT  Torrington   \n",
       "535494                      760            CT  Torrington   \n",
       "535495                      760            CT  Torrington   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30              True         False             True   \n",
       "1                        20             False         False            False   \n",
       "4                        30              True         False             True   \n",
       "5                        10             False         False            False   \n",
       "7                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "535491                   80             False         False            False   \n",
       "535492                   80             False         False            False   \n",
       "535493                   15             False         False            False   \n",
       "535494                   15             False         False            False   \n",
       "535495                   15             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "535491                       Gas  \n",
       "535492                       Gas  \n",
       "535493                       Gas  \n",
       "535494                       Gas  \n",
       "535495                       Gas  \n",
       "\n",
       "[487965 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/CT/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/CT-day/2020-dec-day-CT.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dce9e39382dc10df015dc1225d61261879f8b3c6</td>\n",
       "      <td>2020-12-26 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>761</td>\n",
       "      <td>657</td>\n",
       "      <td>CT</td>\n",
       "      <td>Prospect</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f4091c17f6d7dfa998f53a3e0159dbb214355c7f</td>\n",
       "      <td>2020-12-03 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>CT</td>\n",
       "      <td>Enfield</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>564c6aec11077a013c08daef6f741cea54a99227</td>\n",
       "      <td>2020-12-20 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>703</td>\n",
       "      <td>703</td>\n",
       "      <td>CT</td>\n",
       "      <td>Southbury</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ced1ff247136fb45b6f827deb5cb0c894d162471</td>\n",
       "      <td>2020-12-07 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>CT</td>\n",
       "      <td>Prospect</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dd42deba452d2cea8e2ddb632972e864194a15b9</td>\n",
       "      <td>2020-12-08 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>648</td>\n",
       "      <td>650</td>\n",
       "      <td>644</td>\n",
       "      <td>CT</td>\n",
       "      <td>Madison</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507855</th>\n",
       "      <td>880c33e614e002f07f231be13c494d208eccd723</td>\n",
       "      <td>2020-12-04 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Hamden</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507856</th>\n",
       "      <td>5b1787be6399cd216b5d7b415423412db93e0395</td>\n",
       "      <td>2020-12-02 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507857</th>\n",
       "      <td>5b1787be6399cd216b5d7b415423412db93e0395</td>\n",
       "      <td>2020-12-02 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507858</th>\n",
       "      <td>5b1787be6399cd216b5d7b415423412db93e0395</td>\n",
       "      <td>2020-12-02 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507859</th>\n",
       "      <td>5b1787be6399cd216b5d7b415423412db93e0395</td>\n",
       "      <td>2020-12-02 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>CT</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>478303 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       dce9e39382dc10df015dc1225d61261879f8b3c6  2020-12-26 17:30:00 UTC   \n",
       "1       f4091c17f6d7dfa998f53a3e0159dbb214355c7f  2020-12-03 17:30:00 UTC   \n",
       "2       564c6aec11077a013c08daef6f741cea54a99227  2020-12-20 12:20:00 UTC   \n",
       "3       ced1ff247136fb45b6f827deb5cb0c894d162471  2020-12-07 17:00:00 UTC   \n",
       "4       dd42deba452d2cea8e2ddb632972e864194a15b9  2020-12-08 15:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "507855  880c33e614e002f07f231be13c494d208eccd723  2020-12-04 14:35:00 UTC   \n",
       "507856  5b1787be6399cd216b5d7b415423412db93e0395  2020-12-02 13:05:00 UTC   \n",
       "507857  5b1787be6399cd216b5d7b415423412db93e0395  2020-12-02 14:45:00 UTC   \n",
       "507858  5b1787be6399cd216b5d7b415423412db93e0395  2020-12-02 13:40:00 UTC   \n",
       "507859  5b1787be6399cd216b5d7b415423412db93e0395  2020-12-02 19:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               685                      761   \n",
       "1          heat          hold               700                      691   \n",
       "2          heat          hold               704                      703   \n",
       "3          heat          hold               689                      685   \n",
       "4          heat          auto               648                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "507855     heat          auto               700                      760   \n",
       "507856     heat          hold               698                      760   \n",
       "507857     heat          hold               711                      760   \n",
       "507858     heat          hold               706                      760   \n",
       "507859     heat          hold               712                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           657            CT   Prospect                   65   \n",
       "1                           691            CT    Enfield                   60   \n",
       "2                           703            CT  Southbury                   40   \n",
       "3                           685            CT   Prospect                   65   \n",
       "4                           644            CT    Madison                   50   \n",
       "...                         ...           ...        ...                  ...   \n",
       "507855                      760            CT     Hamden                   50   \n",
       "507856                      760            CT  Fairfield                   90   \n",
       "507857                      760            CT  Fairfield                   90   \n",
       "507858                      760            CT  Fairfield                   90   \n",
       "507859                      760            CT  Fairfield                   90   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                   True         False             True   \n",
       "...                  ...           ...              ...   \n",
       "507855              True         False            False   \n",
       "507856             False         False            False   \n",
       "507857             False         False            False   \n",
       "507858             False         False            False   \n",
       "507859             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "507855                       Gas  \n",
       "507856                       Gas  \n",
       "507857                       Gas  \n",
       "507858                       Gas  \n",
       "507859                       Gas  \n",
       "\n",
       "[478303 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/CT/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/CT/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>007f9b37f1ac5cfe9806f9a1a47809855e7e31b8</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Windsor</td>\n",
       "      <td>648.299346</td>\n",
       "      <td>650.928105</td>\n",
       "      <td>649.856863</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>011dae903f5512427598b14bbb3f02baa8813745</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>637.965116</td>\n",
       "      <td>779.924419</td>\n",
       "      <td>620.148256</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>011dae903f5512427598b14bbb3f02baa8813745</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>660.357143</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01440e0a70900810dceaace98a7965a75bc818dc</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>651.019200</td>\n",
       "      <td>702.430400</td>\n",
       "      <td>648.147520</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01440e0a70900810dceaace98a7965a75bc818dc</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>672.093750</td>\n",
       "      <td>698.343750</td>\n",
       "      <td>697.718750</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>fdce2932511e0bf91c9516b4f56bba7ca424f147</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>669.600000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>697</th>\n",
       "      <td>fdce2932511e0bf91c9516b4f56bba7ca424f147</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>668.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>678.500000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>625.842541</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>616.223757</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>692.695067</td>\n",
       "      <td>693.360987</td>\n",
       "      <td>693.360987</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>700</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>695.704770</td>\n",
       "      <td>691.092709</td>\n",
       "      <td>691.092709</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2810 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    007f9b37f1ac5cfe9806f9a1a47809855e7e31b8   dec  2017     heat   \n",
       "1    011dae903f5512427598b14bbb3f02baa8813745   dec  2017     auto   \n",
       "2    011dae903f5512427598b14bbb3f02baa8813745   dec  2017     auto   \n",
       "3    01440e0a70900810dceaace98a7965a75bc818dc   dec  2017     heat   \n",
       "4    01440e0a70900810dceaace98a7965a75bc818dc   dec  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "696  fdce2932511e0bf91c9516b4f56bba7ca424f147   dec  2020     auto   \n",
       "697  fdce2932511e0bf91c9516b4f56bba7ca424f147   dec  2020     auto   \n",
       "698  fef94e274991646e0f28d91d1aaea3cd709d8825   dec  2020     heat   \n",
       "699  ff498c11fdbb5d698084e0f03a29b69e76ee8566   dec  2020     heat   \n",
       "700  ff498c11fdbb5d698084e0f03a29b69e76ee8566   dec  2020     heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold  South Windsor            648.299346   \n",
       "1            auto        Shelton            637.965116   \n",
       "2            hold        Shelton            660.357143   \n",
       "3            auto       Guilford            651.019200   \n",
       "4            hold       Guilford            672.093750   \n",
       "..            ...            ...                   ...   \n",
       "696          auto        Shelton            669.600000   \n",
       "697          hold        Shelton            668.000000   \n",
       "698          hold     New Canaan            625.842541   \n",
       "699          auto    North Haven            692.695067   \n",
       "700          hold    North Haven            695.704770   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     650.928105                   649.856863   \n",
       "1                     779.924419                   620.148256   \n",
       "2                     760.000000                   660.000000   \n",
       "3                     702.430400                   648.147520   \n",
       "4                     698.343750                   697.718750   \n",
       "..                           ...                          ...   \n",
       "696                   760.000000                   640.000000   \n",
       "697                   750.000000                   678.500000   \n",
       "698                   650.000000                   616.223757   \n",
       "699                   693.360987                   693.360987   \n",
       "700                   691.092709                   691.092709   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                    5.0             False         False            False  \n",
       "2                    5.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                    0.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "696                  5.0             False         False            False  \n",
       "697                  5.0             False         False            False  \n",
       "698                  0.0             False         False            False  \n",
       "699                 70.0             False         False            False  \n",
       "700                 70.0             False         False            False  \n",
       "\n",
       "[2810 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/CT/dec/\" + file)\n",
    "    CT_dec = pd.concat([CT_dec, df])\n",
    "    \n",
    "CT_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_dec.to_csv(\"Scraper_Output/State_Month_Day/CT/CT_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/CT/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>007f9b37f1ac5cfe9806f9a1a47809855e7e31b8</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Windsor</td>\n",
       "      <td>720.473373</td>\n",
       "      <td>720.727811</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>720.490196</td>\n",
       "      <td>718.400000</td>\n",
       "      <td>650.200000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0094e2c467552f4b15765889040931605fb22890</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwalk</td>\n",
       "      <td>721.850000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>27.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>011dae903f5512427598b14bbb3f02baa8813745</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>784.868922</td>\n",
       "      <td>780.002114</td>\n",
       "      <td>620.002114</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01440e0a70900810dceaace98a7965a75bc818dc</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Guilford</td>\n",
       "      <td>727.825439</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2738</th>\n",
       "      <td>fc58d08fd7038c94b9cfcb3287a2bbb6e095745f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>711.339921</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2739</th>\n",
       "      <td>fc92f3d7ab046b1e0482835f5cf10e18372dc434</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairfield</td>\n",
       "      <td>694.968354</td>\n",
       "      <td>687.664557</td>\n",
       "      <td>687.145570</td>\n",
       "      <td>40.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2740</th>\n",
       "      <td>fef94e274991646e0f28d91d1aaea3cd709d8825</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>New Canaan</td>\n",
       "      <td>650.846603</td>\n",
       "      <td>647.909746</td>\n",
       "      <td>647.802334</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2741</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>715.425000</td>\n",
       "      <td>722.812500</td>\n",
       "      <td>722.812500</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2742</th>\n",
       "      <td>ff498c11fdbb5d698084e0f03a29b69e76ee8566</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Haven</td>\n",
       "      <td>699.350000</td>\n",
       "      <td>699.500000</td>\n",
       "      <td>699.500000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15665 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     007f9b37f1ac5cfe9806f9a1a47809855e7e31b8   aug  2017     cool   \n",
       "1     0094e2c467552f4b15765889040931605fb22890   aug  2017     cool   \n",
       "2     0094e2c467552f4b15765889040931605fb22890   aug  2017     cool   \n",
       "3     011dae903f5512427598b14bbb3f02baa8813745   aug  2017     auto   \n",
       "4     01440e0a70900810dceaace98a7965a75bc818dc   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2738  fc58d08fd7038c94b9cfcb3287a2bbb6e095745f   jun  2021     cool   \n",
       "2739  fc92f3d7ab046b1e0482835f5cf10e18372dc434   jun  2021     cool   \n",
       "2740  fef94e274991646e0f28d91d1aaea3cd709d8825   jun  2021     cool   \n",
       "2741  ff498c11fdbb5d698084e0f03a29b69e76ee8566   jun  2021     cool   \n",
       "2742  ff498c11fdbb5d698084e0f03a29b69e76ee8566   jun  2021     heat   \n",
       "\n",
       "     CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0             hold  South Windsor            720.473373   \n",
       "1             auto        Norwalk            720.490196   \n",
       "2             hold        Norwalk            721.850000   \n",
       "3             auto        Shelton            784.868922   \n",
       "4             auto       Guilford            727.825439   \n",
       "...            ...            ...                   ...   \n",
       "2738          hold        Shelton            711.339921   \n",
       "2739          hold      Fairfield            694.968354   \n",
       "2740          hold     New Canaan            650.846603   \n",
       "2741          hold    North Haven            715.425000   \n",
       "2742          hold    North Haven            699.350000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      720.727811                   660.000000   \n",
       "1                      718.400000                   650.200000   \n",
       "2                      710.000000                   710.000000   \n",
       "3                      780.002114                   620.002114   \n",
       "4                      740.000000                   730.000000   \n",
       "...                           ...                          ...   \n",
       "2738                   710.000000                   710.000000   \n",
       "2739                   687.664557                   687.145570   \n",
       "2740                   647.909746                   647.802334   \n",
       "2741                   722.812500                   722.812500   \n",
       "2742                   699.500000                   699.500000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    20.0             False         False            False  \n",
       "1                    27.0             False         False            False  \n",
       "2                    27.0             False         False            False  \n",
       "3                     5.0             False         False            False  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2738                 25.0             False         False            False  \n",
       "2739                 40.0              True         False             True  \n",
       "2740                  0.0             False         False            False  \n",
       "2741                 70.0             False         False            False  \n",
       "2742                 70.0             False         False            False  \n",
       "\n",
       "[15665 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "CT_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/CT/\" + file)\n",
    "    CT_all = pd.concat([CT_all, df])\n",
    "    \n",
    "CT_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "CT_all.to_csv(\"Scraper_Output/State_Month_Day/CT_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['CT']\n",
      "Unique jan_2018: ['CT']\n",
      "Unique jan_2019: ['CT']\n",
      "Unique jan_2020: ['CT']\n",
      "Unique jan_2021: ['CT']\n",
      "Unique feb_2017: ['CT']\n",
      "Unique feb_2018: ['CT']\n",
      "Unique feb_2019: ['CT']\n",
      "Unique feb_2020: ['CT']\n",
      "Unique feb_2021: ['CT']\n",
      "Unique jun_2017: ['CT']\n",
      "Unique jun_2018: ['CT']\n",
      "Unique jun_2019: ['CT']\n",
      "Unique jun_2020: ['CT']\n",
      "Unique jun_2021: ['CT']\n",
      "Unique jul_2017: ['CT']\n",
      "Unique jul_2018: ['CT']\n",
      "Unique jul_2019: ['CT']\n",
      "Unique jul_2020: ['CT']\n",
      "Unique jul_2021: ['CT']\n",
      "Unique aug_2017: ['CT']\n",
      "Unique aug_2018: ['CT']\n",
      "Unique aug_2019: ['CT']\n",
      "Unique aug_2020: ['CT']\n",
      "Unique dec_2017: ['CT']\n",
      "Unique dec_2018: ['CT']\n",
      "Unique dec_2019: ['CT']\n",
      "Unique dec_2020: ['CT']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mCTe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}