DYD_MA-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/MA-day/2017-jan-day-MA.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49e2f121d20e33a2cc4f4d7177360cbba5fbf11b</td>\n",
       "      <td>2017-01-07 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>645</td>\n",
       "      <td>730</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wilbraham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cad0b666683c9a4f4561a2e95f2c485e9ded01a0</td>\n",
       "      <td>2017-01-06 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e5a996e4772f2715b33bbbf3f623cae7c06ab044</td>\n",
       "      <td>2017-01-02 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>648</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>MA</td>\n",
       "      <td>Dedham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e7ffed15c0d31087ae97f7937ebf6f77ee49c0e5</td>\n",
       "      <td>2017-01-07 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>755</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>pepperell</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8a8e0c6771f8f2022cc384fa50ef446822293ffb</td>\n",
       "      <td>2017-01-12 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Somerset</td>\n",
       "      <td>36</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268867</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-01-16 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>630</td>\n",
       "      <td>680</td>\n",
       "      <td>640</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268868</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-01-04 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>639</td>\n",
       "      <td>710</td>\n",
       "      <td>640</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268869</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-01-20 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>629</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268870</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-01-18 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>634</td>\n",
       "      <td>680</td>\n",
       "      <td>640</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>268871</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-01-11 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>680</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>257424 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       49e2f121d20e33a2cc4f4d7177360cbba5fbf11b  2017-01-07 18:55:00 UTC   \n",
       "1       cad0b666683c9a4f4561a2e95f2c485e9ded01a0  2017-01-06 19:20:00 UTC   \n",
       "2       e5a996e4772f2715b33bbbf3f623cae7c06ab044  2017-01-02 13:50:00 UTC   \n",
       "3       e7ffed15c0d31087ae97f7937ebf6f77ee49c0e5  2017-01-07 19:35:00 UTC   \n",
       "4       8a8e0c6771f8f2022cc384fa50ef446822293ffb  2017-01-12 18:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "268867  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-01-16 13:25:00 UTC   \n",
       "268868  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-01-04 14:30:00 UTC   \n",
       "268869  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-01-20 13:50:00 UTC   \n",
       "268870  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-01-18 15:55:00 UTC   \n",
       "268871  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-01-11 19:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               645                      730   \n",
       "1          heat          hold               698                      700   \n",
       "2          heat          auto               648                      695   \n",
       "3          auto          hold               693                      755   \n",
       "4          heat          hold               681                      680   \n",
       "...         ...           ...               ...                      ...   \n",
       "268867     heat          auto               630                      680   \n",
       "268868     heat          auto               639                      710   \n",
       "268869     heat          hold               629                      680   \n",
       "268870     heat          auto               634                      680   \n",
       "268871     heat          hold               645                      680   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           650            MA  Wilbraham                    0   \n",
       "1                           700            MA  Arlington                   35   \n",
       "2                           645            MA     Dedham                    5   \n",
       "3                           695            MA  pepperell                   20   \n",
       "4                           680            MA   Somerset                   36   \n",
       "...                         ...           ...        ...                  ...   \n",
       "268867                      640            MA   Westwood                   25   \n",
       "268868                      640            MA   Westwood                   25   \n",
       "268869                      630            MA   Westwood                   25   \n",
       "268870                      640            MA   Westwood                   25   \n",
       "268871                      650            MA   Westwood                   25   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "268867             False         False            False   \n",
       "268868             False         False            False   \n",
       "268869             False         False            False   \n",
       "268870             False         False            False   \n",
       "268871             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "268867                       Gas  \n",
       "268868                       Gas  \n",
       "268869                       Gas  \n",
       "268870                       Gas  \n",
       "268871                       Gas  \n",
       "\n",
       "[257424 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">021bc49d2737cfc5291795252754c8c84d99fb90</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Plainville</th>\n",
       "      <td>700.295950</td>\n",
       "      <td>700.917445</td>\n",
       "      <td>700.819315</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Plainville</th>\n",
       "      <td>693.263072</td>\n",
       "      <td>691.400327</td>\n",
       "      <td>691.400327</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02382555cf0333ff1ad92861205814fd74616f88</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>South Hadley</th>\n",
       "      <td>636.406250</td>\n",
       "      <td>760.062500</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Boston</th>\n",
       "      <td>665.221311</td>\n",
       "      <td>719.401639</td>\n",
       "      <td>659.565574</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Boston</th>\n",
       "      <td>658.119048</td>\n",
       "      <td>789.285714</td>\n",
       "      <td>638.190476</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fd69d9ea972e1fc7ccea1786f2732babc3387e04</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Littleton</th>\n",
       "      <td>706.700000</td>\n",
       "      <td>720.260000</td>\n",
       "      <td>720.260000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ff30499eeb6176cf4df942531d3cd4ef45a94114</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Northborough</th>\n",
       "      <td>688.056604</td>\n",
       "      <td>686.150943</td>\n",
       "      <td>686.150943</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fff0787bec7315864a164645420fade19da92725</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Agawam</th>\n",
       "      <td>655.524590</td>\n",
       "      <td>817.327869</td>\n",
       "      <td>641.344262</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Agawam</th>\n",
       "      <td>675.060976</td>\n",
       "      <td>685.243902</td>\n",
       "      <td>685.032520</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fffa05184379d65eec4deeb00b14374669d0ff1e</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Salem</th>\n",
       "      <td>691.231726</td>\n",
       "      <td>685.035770</td>\n",
       "      <td>684.643857</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>627 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                         Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville              700.295950   \n",
       "                                                             hold          Plainville              693.263072   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley            636.406250   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                  665.221311   \n",
       "                                                             hold          Boston                  658.119048   \n",
       "...                                                                                                       ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton               706.700000   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough            688.056604   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                  655.524590   \n",
       "                                                             hold          Agawam                  675.060976   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                   691.231726   \n",
       "\n",
       "                                                                                         TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville                     700.917445   \n",
       "                                                             hold          Plainville                     691.400327   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley                   760.062500   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                         719.401639   \n",
       "                                                             hold          Boston                         789.285714   \n",
       "...                                                                                                              ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton                      720.260000   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough                   686.150943   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                         817.327869   \n",
       "                                                             hold          Agawam                         685.243902   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                          685.035770   \n",
       "\n",
       "                                                                                         TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville                     700.819315   \n",
       "                                                             hold          Plainville                     691.400327   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley                   640.000000   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                         659.565574   \n",
       "                                                             hold          Boston                         638.190476   \n",
       "...                                                                                                              ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton                      720.260000   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough                   686.150943   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                         641.344262   \n",
       "                                                             hold          Agawam                         685.032520   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                          684.643857   \n",
       "\n",
       "                                                                                         Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville                   35.0   \n",
       "                                                             hold          Plainville                   35.0   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley                 55.0   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                      100.0   \n",
       "                                                             hold          Boston                      100.0   \n",
       "...                                                                                                      ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton                     5.0   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough                 25.0   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                       95.0   \n",
       "                                                             hold          Agawam                       95.0   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                         6.0   \n",
       "\n",
       "                                                                                         allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                             \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville               False   \n",
       "                                                             hold          Plainville               False   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley             False   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                   False   \n",
       "                                                             hold          Boston                   False   \n",
       "...                                                                                                   ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton                False   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough              True   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                   False   \n",
       "                                                             hold          Agawam                   False   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                    False   \n",
       "\n",
       "                                                                                         Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                         \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville           False   \n",
       "                                                             hold          Plainville           False   \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley         False   \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston               False   \n",
       "                                                             hold          Boston               False   \n",
       "...                                                                                               ...   \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton            False   \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough         False   \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam               False   \n",
       "                                                             hold          Agawam               False   \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                False   \n",
       "\n",
       "                                                                                         Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                           \n",
       "021bc49d2737cfc5291795252754c8c84d99fb90 Jan   2017 heat     auto          Plainville              False  \n",
       "                                                             hold          Plainville              False  \n",
       "02382555cf0333ff1ad92861205814fd74616f88 Jan   2017 heat     hold          South Hadley            False  \n",
       "02fe14bbcc60780d648cd5eae6868f1ff7e79b3c Jan   2017 auto     auto          Boston                  False  \n",
       "                                                             hold          Boston                  False  \n",
       "...                                                                                                  ...  \n",
       "fd69d9ea972e1fc7ccea1786f2732babc3387e04 Jan   2017 heat     hold          Littleton               False  \n",
       "ff30499eeb6176cf4df942531d3cd4ef45a94114 Jan   2017 heat     hold          Northborough            False  \n",
       "fff0787bec7315864a164645420fade19da92725 Jan   2017 heat     auto          Agawam                  False  \n",
       "                                                             hold          Agawam                  False  \n",
       "fffa05184379d65eec4deeb00b14374669d0ff1e Jan   2017 heat     hold          Salem                   False  \n",
       "\n",
       "[627 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/MA/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/MA-day/2018-jan-day-MA.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f0caa2bb680030a496edc7f4daee22bec88c1f79</td>\n",
       "      <td>2018-01-19 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>789</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MA</td>\n",
       "      <td>Braintree</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6d5ca0833d3bcf140823ebc1dd85b09f5e56fc03</td>\n",
       "      <td>2018-01-05 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>797</td>\n",
       "      <td>707</td>\n",
       "      <td>MA</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fe2bdeb6fe92670a4befad74d25a7061473e51f8</td>\n",
       "      <td>2018-01-11 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>755</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Norwood</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>056acf9edbb36501bae8b3a2f0031a4874f7448f</td>\n",
       "      <td>2018-01-25 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Attleboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9382a25091817667a8401669710e5f48956b6f56</td>\n",
       "      <td>2018-01-04 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846143</th>\n",
       "      <td>946a303b7e7362e3045135a21363b1230adbb8ef</td>\n",
       "      <td>2018-01-23 11:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>616</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>Norwood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846144</th>\n",
       "      <td>bbfa7dc7930d25700f7499bb163714c2640e5c8a</td>\n",
       "      <td>2018-01-12 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846145</th>\n",
       "      <td>b39ebf99bf4bf900fa1c52049187daedfdd7dc96</td>\n",
       "      <td>2018-01-31 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sharon</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846146</th>\n",
       "      <td>c8c3bfb1b8d6ba0aee9b0fb5f277fbb6da0cba32</td>\n",
       "      <td>2018-01-02 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Weston</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846147</th>\n",
       "      <td>1316ae40a733eb2da85f8b55f97353d482de727e</td>\n",
       "      <td>2018-01-04 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Tewksbury</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>784234 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f0caa2bb680030a496edc7f4daee22bec88c1f79  2018-01-19 19:00:00 UTC   \n",
       "1       6d5ca0833d3bcf140823ebc1dd85b09f5e56fc03  2018-01-05 19:00:00 UTC   \n",
       "2       fe2bdeb6fe92670a4befad74d25a7061473e51f8  2018-01-11 12:00:00 UTC   \n",
       "3       056acf9edbb36501bae8b3a2f0031a4874f7448f  2018-01-25 13:00:00 UTC   \n",
       "4       9382a25091817667a8401669710e5f48956b6f56  2018-01-04 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "846143  946a303b7e7362e3045135a21363b1230adbb8ef  2018-01-23 11:20:00 UTC   \n",
       "846144  bbfa7dc7930d25700f7499bb163714c2640e5c8a  2018-01-12 14:00:00 UTC   \n",
       "846145  b39ebf99bf4bf900fa1c52049187daedfdd7dc96  2018-01-31 13:15:00 UTC   \n",
       "846146  c8c3bfb1b8d6ba0aee9b0fb5f277fbb6da0cba32  2018-01-02 15:00:00 UTC   \n",
       "846147  1316ae40a733eb2da85f8b55f97353d482de727e  2018-01-04 13:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               789                      790   \n",
       "1          heat          auto               708                      797   \n",
       "2          auto          hold               710                      755   \n",
       "3          auto          hold               705                      755   \n",
       "4          auto          hold               654                      705   \n",
       "...         ...           ...               ...                      ...   \n",
       "846143     auto          auto               616                      760   \n",
       "846144     auto          hold               688                      760   \n",
       "846145     auto          hold               679                      760   \n",
       "846146     auto          hold               703                      760   \n",
       "846147     heat          auto               710                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           790            MA  Braintree                   10   \n",
       "1                           707            MA  Watertown                  105   \n",
       "2                           685            MA    Norwood                   67   \n",
       "3                           705            MA  Attleboro                    0   \n",
       "4                           655            MA     Boston                    0   \n",
       "...                         ...           ...        ...                  ...   \n",
       "846143                      620            MA    Norwood                    0   \n",
       "846144                      690            MA       Lynn                   10   \n",
       "846145                      700            MA     Sharon                   35   \n",
       "846146                      710            MA     Weston                    5   \n",
       "846147                      710            MA  Tewksbury                   29   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "846143             False         False            False   \n",
       "846144             False         False            False   \n",
       "846145             False         False            False   \n",
       "846146             False         False            False   \n",
       "846147              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "846143                       Gas  \n",
       "846144                       Gas  \n",
       "846145                       Gas  \n",
       "846146                       Gas  \n",
       "846147                       Gas  \n",
       "\n",
       "[784234 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/MA/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/MA-day/2019-jan-day-MA.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2019-01-11 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>703</td>\n",
       "      <td>703</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3f07d74c917482309858daaeeba7f3d15c14592e</td>\n",
       "      <td>2019-01-19 12:55:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2019-01-24 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9bdb317eb4283f890621a591754b7b268a991756</td>\n",
       "      <td>2019-01-03 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>644</td>\n",
       "      <td>669</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>billerica</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3eab3d29baa95ecbbc6ca60d2da1f71bcef788a6</td>\n",
       "      <td>2019-01-21 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332564</th>\n",
       "      <td>4b395f86a687a1646dec53fdb9d642c676f3d87b</td>\n",
       "      <td>2019-01-11 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Neighborhood Nine</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332565</th>\n",
       "      <td>9c012431edb2d215a0b9f419ba7589fb90357d9e</td>\n",
       "      <td>2019-01-22 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Indian Orchard</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332566</th>\n",
       "      <td>c3c47b13f19b003a8b04873ec0fb882a3afed1cb</td>\n",
       "      <td>2019-01-06 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>645</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>Mansfield</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332567</th>\n",
       "      <td>9e560f186f452c50ffb1b63902ad573f282a555a</td>\n",
       "      <td>2019-01-22 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>609</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Chelmsford</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332568</th>\n",
       "      <td>aa8dff072b080ffd3900774539f1a28bff78fb59</td>\n",
       "      <td>2019-01-03 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1190351 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        675e97e14602b3adfee227ef565c0205a1f0d40c  2019-01-11 14:45:00 UTC   \n",
       "1        3f07d74c917482309858daaeeba7f3d15c14592e  2019-01-19 12:55:00 UTC   \n",
       "2        675e97e14602b3adfee227ef565c0205a1f0d40c  2019-01-24 18:00:00 UTC   \n",
       "3        9bdb317eb4283f890621a591754b7b268a991756  2019-01-03 13:35:00 UTC   \n",
       "4        3eab3d29baa95ecbbc6ca60d2da1f71bcef788a6  2019-01-21 12:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1332564  4b395f86a687a1646dec53fdb9d642c676f3d87b  2019-01-11 13:50:00 UTC   \n",
       "1332565  9c012431edb2d215a0b9f419ba7589fb90357d9e  2019-01-22 13:30:00 UTC   \n",
       "1332566  c3c47b13f19b003a8b04873ec0fb882a3afed1cb  2019-01-06 15:25:00 UTC   \n",
       "1332567  9e560f186f452c50ffb1b63902ad573f282a555a  2019-01-22 15:55:00 UTC   \n",
       "1332568  aa8dff072b080ffd3900774539f1a28bff78fb59  2019-01-03 16:00:00 UTC   \n",
       "\n",
       "            HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0               heat          hold               694                      703   \n",
       "1        auxHeatOnly          hold               682                      695   \n",
       "2               heat          hold               692                      698   \n",
       "3               heat          auto               644                      669   \n",
       "4               heat          auto               681                      725   \n",
       "...              ...           ...               ...                      ...   \n",
       "1332564         auto          hold               706                      760   \n",
       "1332565         auto          auto               707                      760   \n",
       "1332566         heat          auto               645                      760   \n",
       "1332567         heat          hold               609                      760   \n",
       "1332568         heat          hold               756                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            703            MA      Yarmouth port   \n",
       "1                            695            MA            Andover   \n",
       "2                            698            MA      Yarmouth port   \n",
       "3                            650            MA         billerica    \n",
       "4                            725            MA      North Andover   \n",
       "...                          ...           ...                ...   \n",
       "1332564                      710            MA  Neighborhood Nine   \n",
       "1332565                      710            MA     Indian Orchard   \n",
       "1332566                      650            MA          Mansfield   \n",
       "1332567                      760            MA   North Chelmsford   \n",
       "1332568                      760            MA           Lawrence   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5             False         False            False   \n",
       "1                         60             False         False             True   \n",
       "2                          5             False         False            False   \n",
       "3                         90             False         False            False   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1332564                    5             False         False            False   \n",
       "1332565                    0             False         False            False   \n",
       "1332566                   20             False         False            False   \n",
       "1332567                   50             False         False            False   \n",
       "1332568                   90             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1332564                       Gas  \n",
       "1332565                       Gas  \n",
       "1332566                       Gas  \n",
       "1332567                       Gas  \n",
       "1332568                       Gas  \n",
       "\n",
       "[1190351 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/MA/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/MA-day/2020-jan-day-MA.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>69b0f4b453a0a73342d4e6ecbca900433a97e793</td>\n",
       "      <td>2020-01-29 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lakeville</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9d30b15f9fcef102c976e772070b4f0028c725bf</td>\n",
       "      <td>2020-01-23 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>668</td>\n",
       "      <td>668</td>\n",
       "      <td>MA</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9885ebe9e6190eed190aea0b902d3331eb4d57d8</td>\n",
       "      <td>2020-01-18 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0e76511b11c6f38465ad968c99410de1dae1682b</td>\n",
       "      <td>2020-01-11 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>64af2451003969012b5faed08ea2f4eff9730697</td>\n",
       "      <td>2020-01-13 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>743</td>\n",
       "      <td>697</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ciudad de M&amp;#x9d;&amp;#x</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338805</th>\n",
       "      <td>fb195d96b05e87c63127f30fca435c90669efcf1</td>\n",
       "      <td>2020-01-16 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>627</td>\n",
       "      <td>760</td>\n",
       "      <td>640</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ayer</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338806</th>\n",
       "      <td>8a00321ec4591c4319cc36112ccf03be0d258c4d</td>\n",
       "      <td>2020-01-18 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338807</th>\n",
       "      <td>f2d8374f370ed0e09e7e5bf83264b4169eb7db78</td>\n",
       "      <td>2020-01-25 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338808</th>\n",
       "      <td>5e8130eba436a420c18643b6f39c31c0ddf350d0</td>\n",
       "      <td>2020-01-17 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Freetown</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338809</th>\n",
       "      <td>4f2bcc28fb4e49fbd2dff4738ce6255793cd79ef</td>\n",
       "      <td>2020-01-19 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>646</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wilbraham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1214250 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        69b0f4b453a0a73342d4e6ecbca900433a97e793  2020-01-29 16:35:00 UTC   \n",
       "1        9d30b15f9fcef102c976e772070b4f0028c725bf  2020-01-23 18:05:00 UTC   \n",
       "2        9885ebe9e6190eed190aea0b902d3331eb4d57d8  2020-01-18 19:45:00 UTC   \n",
       "3        0e76511b11c6f38465ad968c99410de1dae1682b  2020-01-11 17:15:00 UTC   \n",
       "4        64af2451003969012b5faed08ea2f4eff9730697  2020-01-13 19:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1338805  fb195d96b05e87c63127f30fca435c90669efcf1  2020-01-16 13:55:00 UTC   \n",
       "1338806  8a00321ec4591c4319cc36112ccf03be0d258c4d  2020-01-18 14:15:00 UTC   \n",
       "1338807  f2d8374f370ed0e09e7e5bf83264b4169eb7db78  2020-01-25 12:00:00 UTC   \n",
       "1338808  5e8130eba436a420c18643b6f39c31c0ddf350d0  2020-01-17 14:55:00 UTC   \n",
       "1338809  4f2bcc28fb4e49fbd2dff4738ce6255793cd79ef  2020-01-19 13:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               700                      702   \n",
       "1           heat          hold               663                      668   \n",
       "2           heat          auto               699                      735   \n",
       "3           heat          hold               720                      718   \n",
       "4           cool          auto               719                      743   \n",
       "...          ...           ...               ...                      ...   \n",
       "1338805     heat          auto               627                      760   \n",
       "1338806     auto          hold               678                      760   \n",
       "1338807     auto          hold               685                      760   \n",
       "1338808     auto          auto               702                      760   \n",
       "1338809     heat          auto               646                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                  City  \\\n",
       "0                            702            MA             Lakeville   \n",
       "1                            668            MA                Woburn   \n",
       "2                            735            MA                Boston   \n",
       "3                            718            MA               Andover   \n",
       "4                            697            MA  Ciudad de M&#x9d;&#x   \n",
       "...                          ...           ...                   ...   \n",
       "1338805                      640            MA                  Ayer   \n",
       "1338806                      700            MA                Newton   \n",
       "1338807                      680            MA                Natick   \n",
       "1338808                      700            MA              Freetown   \n",
       "1338809                      650            MA             Wilbraham   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False            False   \n",
       "1                         20              True         False             True   \n",
       "2                        120             False         False            False   \n",
       "3                          0              True         False            False   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1338805                   10             False         False            False   \n",
       "1338806                   15             False         False            False   \n",
       "1338807                   10             False         False            False   \n",
       "1338808                    0             False         False            False   \n",
       "1338809                    5             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1338805                       Gas  \n",
       "1338806                       Gas  \n",
       "1338807                       Gas  \n",
       "1338808                       Gas  \n",
       "1338809                       Gas  \n",
       "\n",
       "[1214250 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/MA/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/MA-day/2021-jan-day-MA.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2021-01-14 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>739</td>\n",
       "      <td>689</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7d620630c1106bb25eaca1c85d2c0a4826cdbc9d</td>\n",
       "      <td>2021-01-26 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>724</td>\n",
       "      <td>724</td>\n",
       "      <td>MA</td>\n",
       "      <td>Brockton</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1b644d6cf7869c9a4278704fa04009985f1c5a3d</td>\n",
       "      <td>2021-01-06 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>MA</td>\n",
       "      <td>Bedford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b54b3cb4c02d7a5b617ffbf9ac5bc43f5ec3ee93</td>\n",
       "      <td>2021-01-06 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>MA</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ee4a4331237fb27571c379c9e22c7ea107c8a79d</td>\n",
       "      <td>2021-01-12 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>742</td>\n",
       "      <td>661</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wrentham</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987732</th>\n",
       "      <td>7f0a41db47e813113234753d7d33dec719429f57</td>\n",
       "      <td>2021-01-27 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987733</th>\n",
       "      <td>e08d6dff90ec6e4ba1c64a8bd3a09875617e0bbe</td>\n",
       "      <td>2021-01-28 11:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987734</th>\n",
       "      <td>0d3dd4457fb8476c00d0319206d0e6c368e2bb37</td>\n",
       "      <td>2021-01-27 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Danvers</td>\n",
       "      <td>37</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987735</th>\n",
       "      <td>492784f1c9ee5262dfd9147baf61632e236f02db</td>\n",
       "      <td>2021-01-09 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Malden</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987736</th>\n",
       "      <td>023185350ab8ebc35230d3fcfd2a597d16f5e6c0</td>\n",
       "      <td>2021-01-11 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>925529 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       675e97e14602b3adfee227ef565c0205a1f0d40c  2021-01-14 19:15:00 UTC   \n",
       "1       7d620630c1106bb25eaca1c85d2c0a4826cdbc9d  2021-01-26 14:00:00 UTC   \n",
       "2       1b644d6cf7869c9a4278704fa04009985f1c5a3d  2021-01-06 15:15:00 UTC   \n",
       "3       b54b3cb4c02d7a5b617ffbf9ac5bc43f5ec3ee93  2021-01-06 19:55:00 UTC   \n",
       "4       ee4a4331237fb27571c379c9e22c7ea107c8a79d  2021-01-12 13:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "987732  7f0a41db47e813113234753d7d33dec719429f57  2021-01-27 16:20:00 UTC   \n",
       "987733  e08d6dff90ec6e4ba1c64a8bd3a09875617e0bbe  2021-01-28 11:55:00 UTC   \n",
       "987734  0d3dd4457fb8476c00d0319206d0e6c368e2bb37  2021-01-27 18:45:00 UTC   \n",
       "987735  492784f1c9ee5262dfd9147baf61632e236f02db  2021-01-09 16:25:00 UTC   \n",
       "987736  023185350ab8ebc35230d3fcfd2a597d16f5e6c0  2021-01-11 14:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               688                      739   \n",
       "1          heat          hold               715                      724   \n",
       "2          heat          hold               663                      655   \n",
       "3          heat          hold               659                      665   \n",
       "4          heat          hold               706                      742   \n",
       "...         ...           ...               ...                      ...   \n",
       "987732     auto          hold               696                      760   \n",
       "987733     auto          hold               682                      760   \n",
       "987734     auto          hold               694                      760   \n",
       "987735     cool          hold               715                      760   \n",
       "987736     auto          hold               691                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           689            MA  Yarmouth port   \n",
       "1                           724            MA       Brockton   \n",
       "2                           655            MA        Bedford   \n",
       "3                           665            MA         Woburn   \n",
       "4                           661            MA       Wrentham   \n",
       "...                         ...           ...            ...   \n",
       "987732                      700            MA         Newton   \n",
       "987733                      680            MA      Hopkinton   \n",
       "987734                      700            MA        Danvers   \n",
       "987735                      760            MA         Malden   \n",
       "987736                      700            MA      Hopkinton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        50             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        40             False         False             True   \n",
       "4                        65             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "987732                    0             False         False            False   \n",
       "987733                    9              True         False            False   \n",
       "987734                   37              True         False             True   \n",
       "987735                    5              True         False            False   \n",
       "987736                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "987732                       Gas  \n",
       "987733                       Gas  \n",
       "987734                  Electric  \n",
       "987735                       Gas  \n",
       "987736                       Gas  \n",
       "\n",
       "[925529 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/MA/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>700.295950</td>\n",
       "      <td>700.917445</td>\n",
       "      <td>700.819315</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>693.263072</td>\n",
       "      <td>691.400327</td>\n",
       "      <td>691.400327</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02382555cf0333ff1ad92861205814fd74616f88</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Hadley</td>\n",
       "      <td>636.406250</td>\n",
       "      <td>760.062500</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Boston</td>\n",
       "      <td>665.221311</td>\n",
       "      <td>719.401639</td>\n",
       "      <td>659.565574</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boston</td>\n",
       "      <td>658.119048</td>\n",
       "      <td>789.285714</td>\n",
       "      <td>638.190476</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1286</th>\n",
       "      <td>fe66c6409aa64ed3ed81284c061c7b316b078095</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>712.574468</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1287</th>\n",
       "      <td>fe8a04920bc9f8b674801fa40c92b000ea23264e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medford</td>\n",
       "      <td>667.514865</td>\n",
       "      <td>667.648649</td>\n",
       "      <td>667.279730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1288</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westford</td>\n",
       "      <td>696.988601</td>\n",
       "      <td>698.922280</td>\n",
       "      <td>698.922280</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1289</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>680.275641</td>\n",
       "      <td>690.314103</td>\n",
       "      <td>689.955128</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1290</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>698.136680</td>\n",
       "      <td>703.307336</td>\n",
       "      <td>703.216216</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8149 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     021bc49d2737cfc5291795252754c8c84d99fb90   Jan  2017     heat   \n",
       "1     021bc49d2737cfc5291795252754c8c84d99fb90   Jan  2017     heat   \n",
       "2     02382555cf0333ff1ad92861205814fd74616f88   Jan  2017     heat   \n",
       "3     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   Jan  2017     auto   \n",
       "4     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   Jan  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1286  fe66c6409aa64ed3ed81284c061c7b316b078095   Jan  2021     heat   \n",
       "1287  fe8a04920bc9f8b674801fa40c92b000ea23264e   Jan  2021     heat   \n",
       "1288  feb0e1fc76a69b0d234d4874cf27bae767b20c81   Jan  2021     heat   \n",
       "1289  fff0787bec7315864a164645420fade19da92725   Jan  2021     heat   \n",
       "1290  fffa05184379d65eec4deeb00b14374669d0ff1e   Jan  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto    Plainville            700.295950   \n",
       "1             hold    Plainville            693.263072   \n",
       "2             hold  South Hadley            636.406250   \n",
       "3             auto        Boston            665.221311   \n",
       "4             hold        Boston            658.119048   \n",
       "...            ...           ...                   ...   \n",
       "1286          hold     Cambridge            712.574468   \n",
       "1287          hold       Medford            667.514865   \n",
       "1288          hold      Westford            696.988601   \n",
       "1289          hold        Agawam            680.275641   \n",
       "1290          hold         Salem            698.136680   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      700.917445                   700.819315   \n",
       "1                      691.400327                   691.400327   \n",
       "2                      760.062500                   640.000000   \n",
       "3                      719.401639                   659.565574   \n",
       "4                      789.285714                   638.190476   \n",
       "...                           ...                          ...   \n",
       "1286                   730.000000                   730.000000   \n",
       "1287                   667.648649                   667.279730   \n",
       "1288                   698.922280                   698.922280   \n",
       "1289                   690.314103                   689.955128   \n",
       "1290                   703.307336                   703.216216   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    35.0             False         False            False  \n",
       "1                    35.0             False         False            False  \n",
       "2                    55.0             False         False            False  \n",
       "3                   100.0             False         False            False  \n",
       "4                   100.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1286                  0.0              True         False            False  \n",
       "1287                  0.0             False         False            False  \n",
       "1288                 30.0              True         False            False  \n",
       "1289                 95.0             False         False            False  \n",
       "1290                  6.0             False         False            False  \n",
       "\n",
       "[8149 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/jan/\" + file)\n",
    "    MA_jan = pd.concat([MA_jan, df])\n",
    "    \n",
    "MA_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_jan.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/MA-day/2017-feb-day-MA.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c562dffa30de9f88bb70f7eff3d10b3162fff7b</td>\n",
       "      <td>2017-02-10 11:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>595</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Northborough</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47ffbea1156daac92a75ac7c1ddde6c2044c7f15</td>\n",
       "      <td>2017-02-06 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Methuen</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>08f4bbffd98229b9dff6e093b629feb4779fdf89</td>\n",
       "      <td>2017-02-05 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>770</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>Arlington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78918209642ca8aa6b99abc2ee85fea48f5d495f</td>\n",
       "      <td>2017-02-05 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lowell</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e34a0e8914b20c92128bd185cbd2b06f645f8819</td>\n",
       "      <td>2017-02-08 11:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>770</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westford</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261555</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-02-02 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>627</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261556</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-02-26 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>622</td>\n",
       "      <td>680</td>\n",
       "      <td>610</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261557</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-02-23 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>680</td>\n",
       "      <td>610</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261558</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-02-27 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>630</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261559</th>\n",
       "      <td>fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b</td>\n",
       "      <td>2017-02-15 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>631</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>244872 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c562dffa30de9f88bb70f7eff3d10b3162fff7b  2017-02-10 11:45:00 UTC   \n",
       "1       47ffbea1156daac92a75ac7c1ddde6c2044c7f15  2017-02-06 13:05:00 UTC   \n",
       "2       08f4bbffd98229b9dff6e093b629feb4779fdf89  2017-02-05 17:00:00 UTC   \n",
       "3       78918209642ca8aa6b99abc2ee85fea48f5d495f  2017-02-05 19:55:00 UTC   \n",
       "4       e34a0e8914b20c92128bd185cbd2b06f645f8819  2017-02-08 11:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "261555  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-02-02 13:55:00 UTC   \n",
       "261556  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-02-26 18:35:00 UTC   \n",
       "261557  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-02-23 19:25:00 UTC   \n",
       "261558  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-02-27 18:40:00 UTC   \n",
       "261559  fd0e7ab78e129ce6080a9c3397a0e03bbda2be9b  2017-02-15 18:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               595                      770   \n",
       "1          heat          auto               711                      680   \n",
       "2          auto          hold               711                      770   \n",
       "3          heat          auto               710                      710   \n",
       "4          auto          hold               720                      770   \n",
       "...         ...           ...               ...                      ...   \n",
       "261555     heat          hold               627                      680   \n",
       "261556     heat          hold               622                      680   \n",
       "261557     heat          hold               653                      680   \n",
       "261558     heat          hold               630                      680   \n",
       "261559     heat          hold               631                      680   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           700            MA  Northborough   \n",
       "1                           680            MA       Methuen   \n",
       "2                           720            MA     Arlington   \n",
       "3                           710            MA        Lowell   \n",
       "4                           720            MA      Westford   \n",
       "...                         ...           ...           ...   \n",
       "261555                      630            MA      Westwood   \n",
       "261556                      610            MA      Westwood   \n",
       "261557                      610            MA      Westwood   \n",
       "261558                      630            MA      Westwood   \n",
       "261559                      630            MA      Westwood   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        29              True         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False             True   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "261555                   25             False         False            False   \n",
       "261556                   25             False         False            False   \n",
       "261557                   25             False         False            False   \n",
       "261558                   25             False         False            False   \n",
       "261559                   25             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "261555                       Gas  \n",
       "261556                       Gas  \n",
       "261557                       Gas  \n",
       "261558                       Gas  \n",
       "261559                       Gas  \n",
       "\n",
       "[244872 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/MA/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/MA-day/2018-feb-day-MA.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>467cb7943b8615a52634e7903dcad575575e78d7</td>\n",
       "      <td>2018-02-16 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>790</td>\n",
       "      <td>670</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Yarmouth</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>83d1cf93fa2f36bab79c5321d3cceccd291c6c80</td>\n",
       "      <td>2018-02-12 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Abington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>640848a9a73a5e07755ab9e2bdc01eb4bdfeb119</td>\n",
       "      <td>2018-02-11 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>721</td>\n",
       "      <td>734</td>\n",
       "      <td>678</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Springfield</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>46d69c27b5161ac09e4cd52944536c28549542c5</td>\n",
       "      <td>2018-02-08 09:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>734</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>64e6427cf9438a69c43bc6166a69f67988f35989</td>\n",
       "      <td>2018-02-26 12:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>777</td>\n",
       "      <td>617</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749528</th>\n",
       "      <td>8c12084866a84a3b7c9c56a3753b839b29088cd9</td>\n",
       "      <td>2018-02-17 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Brookline</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749529</th>\n",
       "      <td>b39ebf99bf4bf900fa1c52049187daedfdd7dc96</td>\n",
       "      <td>2018-02-07 10:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sharon</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749530</th>\n",
       "      <td>bbfa7dc7930d25700f7499bb163714c2640e5c8a</td>\n",
       "      <td>2018-02-03 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749531</th>\n",
       "      <td>e023c1e547bf7db96e001007be7b1dd244f418ff</td>\n",
       "      <td>2018-02-17 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>MA</td>\n",
       "      <td>Waltham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749532</th>\n",
       "      <td>9d76a699cacdd33fb2279155f5059f320bd8202e</td>\n",
       "      <td>2018-02-27 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sudbury</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>674196 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       467cb7943b8615a52634e7903dcad575575e78d7  2018-02-16 17:10:00 UTC   \n",
       "2       83d1cf93fa2f36bab79c5321d3cceccd291c6c80  2018-02-12 14:10:00 UTC   \n",
       "3       640848a9a73a5e07755ab9e2bdc01eb4bdfeb119  2018-02-11 15:30:00 UTC   \n",
       "4       46d69c27b5161ac09e4cd52944536c28549542c5  2018-02-08 09:45:00 UTC   \n",
       "5       64e6427cf9438a69c43bc6166a69f67988f35989  2018-02-26 12:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "749528  8c12084866a84a3b7c9c56a3753b839b29088cd9  2018-02-17 18:25:00 UTC   \n",
       "749529  b39ebf99bf4bf900fa1c52049187daedfdd7dc96  2018-02-07 10:50:00 UTC   \n",
       "749530  bbfa7dc7930d25700f7499bb163714c2640e5c8a  2018-02-03 17:45:00 UTC   \n",
       "749531  e023c1e547bf7db96e001007be7b1dd244f418ff  2018-02-17 19:10:00 UTC   \n",
       "749532  9d76a699cacdd33fb2279155f5059f320bd8202e  2018-02-27 18:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               663                      790   \n",
       "2          heat          hold               693                      695   \n",
       "3          heat          auto               721                      734   \n",
       "4          heat          hold               692                      734   \n",
       "5          heat          hold               709                      777   \n",
       "...         ...           ...               ...                      ...   \n",
       "749528     auto          hold               691                      760   \n",
       "749529     auto          hold               689                      760   \n",
       "749530     auto          auto               687                      760   \n",
       "749531     heat          auto               743                      760   \n",
       "749532     auto          hold               701                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                           670            MA     West Yarmouth   \n",
       "2                           695            MA          Abington   \n",
       "3                           678            MA  West Springfield   \n",
       "4                           698            MA           Ashland   \n",
       "5                           617            MA         Cambridge   \n",
       "...                         ...           ...               ...   \n",
       "749528                      680            MA         Brookline   \n",
       "749529                      700            MA            Sharon   \n",
       "749530                      690            MA              Lynn   \n",
       "749531                      650            MA           Waltham   \n",
       "749532                      690            MA           Sudbury   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        47             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        37             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "5                       120             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "749528                   20              True         False             True   \n",
       "749529                   35             False         False            False   \n",
       "749530                   10             False         False            False   \n",
       "749531                    5             False         False            False   \n",
       "749532                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "749528                  Electric  \n",
       "749529                       Gas  \n",
       "749530                       Gas  \n",
       "749531                       Gas  \n",
       "749532                       Gas  \n",
       "\n",
       "[674196 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/MA/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/MA-day/2019-feb-day-MA.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cef8dc73bbc4555fb070240c4e56d310a30349a2</td>\n",
       "      <td>2019-02-27 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>676</td>\n",
       "      <td>676</td>\n",
       "      <td>MA</td>\n",
       "      <td>Hingham</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>81a4095dcfd11fb982255c7e9d1642a64310fafb</td>\n",
       "      <td>2019-02-15 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5fd7a339aedb239a5c5ab12b7530daec40db521e</td>\n",
       "      <td>2019-02-02 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>824</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wellesley</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>80f078487b48095f017b75c7389b82f198157d9d</td>\n",
       "      <td>2019-02-21 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>725</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Somerville</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>46d69c27b5161ac09e4cd52944536c28549542c5</td>\n",
       "      <td>2019-02-25 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>734</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914157</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2019-02-19 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914158</th>\n",
       "      <td>467cb7943b8615a52634e7903dcad575575e78d7</td>\n",
       "      <td>2019-02-11 10:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Yarmouth</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914159</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2019-02-23 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914160</th>\n",
       "      <td>467cb7943b8615a52634e7903dcad575575e78d7</td>\n",
       "      <td>2019-02-17 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Yarmouth</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>914161</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2019-02-20 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>823958 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cef8dc73bbc4555fb070240c4e56d310a30349a2  2019-02-27 15:10:00 UTC   \n",
       "1       81a4095dcfd11fb982255c7e9d1642a64310fafb  2019-02-15 13:45:00 UTC   \n",
       "3       5fd7a339aedb239a5c5ab12b7530daec40db521e  2019-02-02 13:00:00 UTC   \n",
       "4       80f078487b48095f017b75c7389b82f198157d9d  2019-02-21 17:50:00 UTC   \n",
       "5       46d69c27b5161ac09e4cd52944536c28549542c5  2019-02-25 14:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "914157  f765cc16f017612fca90aca7449e51aa1d0c8e98  2019-02-19 17:55:00 UTC   \n",
       "914158  467cb7943b8615a52634e7903dcad575575e78d7  2019-02-11 10:15:00 UTC   \n",
       "914159  f765cc16f017612fca90aca7449e51aa1d0c8e98  2019-02-23 18:30:00 UTC   \n",
       "914160  467cb7943b8615a52634e7903dcad575575e78d7  2019-02-17 12:15:00 UTC   \n",
       "914161  f765cc16f017612fca90aca7449e51aa1d0c8e98  2019-02-20 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               678                      676   \n",
       "1          heat          hold               687                      682   \n",
       "3          auto          auto               701                      824   \n",
       "4          heat          auto               688                      725   \n",
       "5          heat          auto               682                      734   \n",
       "...         ...           ...               ...                      ...   \n",
       "914157     auto          hold               705                      760   \n",
       "914158     auto          hold               663                      760   \n",
       "914159     auto          hold               726                      760   \n",
       "914160     auto          hold               667                      760   \n",
       "914161     auto          hold               710                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           676            MA        Hingham   \n",
       "1                           682            MA        Andover   \n",
       "3                           698            MA      Wellesley   \n",
       "4                           690            MA     Somerville   \n",
       "5                           680            MA        Ashland   \n",
       "...                         ...           ...            ...   \n",
       "914157                      700            MA         Boston   \n",
       "914158                      670            MA  West Yarmouth   \n",
       "914159                      710            MA         Boston   \n",
       "914160                      670            MA  West Yarmouth   \n",
       "914161                      700            MA         Boston   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60             False         False            False   \n",
       "1                       100             False         False            False   \n",
       "3                        75             False         False            False   \n",
       "4                        90              True         False            False   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "914157                    0             False         False             True   \n",
       "914158                   47             False         False            False   \n",
       "914159                    0             False         False             True   \n",
       "914160                   47             False         False            False   \n",
       "914161                    0             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "914157                  Electric  \n",
       "914158                       Gas  \n",
       "914159                  Electric  \n",
       "914160                       Gas  \n",
       "914161                  Electric  \n",
       "\n",
       "[823958 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/MA/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/MA-day/2020-feb-day-MA.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ae74a8090db40186913d7c8ad3b75cfb1cb5a714</td>\n",
       "      <td>2020-02-12 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>693</td>\n",
       "      <td>693</td>\n",
       "      <td>MA</td>\n",
       "      <td>Fitchburg</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>265449d76ad9cdbd656baf5f888501f9a6a22009</td>\n",
       "      <td>2020-02-21 13:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Brookfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8c69ce26609b6184721955eb990c2611e60c402c</td>\n",
       "      <td>2020-02-19 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>748</td>\n",
       "      <td>613</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stoughton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1563b6f82ae6503046a17895ddbc6ab82b435ec4</td>\n",
       "      <td>2020-02-10 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Brookline</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d09e1f68990c452fe1afe5d84c55c7c1df9b1bcc</td>\n",
       "      <td>2020-02-19 16:40:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Washington Square</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224234</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2020-02-18 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224235</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2020-02-16 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224236</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2020-02-16 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224238</th>\n",
       "      <td>ae5fb2eda9759f1f17e8cc5b51998d819e712b59</td>\n",
       "      <td>2020-02-22 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>East Falmouth</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224239</th>\n",
       "      <td>faf937a4a11c51e03223c045aeea0cd1464abc64</td>\n",
       "      <td>2020-02-21 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sudbury</td>\n",
       "      <td>57</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1106330 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        ae74a8090db40186913d7c8ad3b75cfb1cb5a714  2020-02-12 12:20:00 UTC   \n",
       "1        265449d76ad9cdbd656baf5f888501f9a6a22009  2020-02-21 13:00:00 UTC   \n",
       "2        8c69ce26609b6184721955eb990c2611e60c402c  2020-02-19 13:55:00 UTC   \n",
       "3        1563b6f82ae6503046a17895ddbc6ab82b435ec4  2020-02-10 16:10:00 UTC   \n",
       "4        d09e1f68990c452fe1afe5d84c55c7c1df9b1bcc  2020-02-19 16:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1224234  f765cc16f017612fca90aca7449e51aa1d0c8e98  2020-02-18 13:40:00 UTC   \n",
       "1224235  675e97e14602b3adfee227ef565c0205a1f0d40c  2020-02-16 13:15:00 UTC   \n",
       "1224236  f765cc16f017612fca90aca7449e51aa1d0c8e98  2020-02-16 14:15:00 UTC   \n",
       "1224238  ae5fb2eda9759f1f17e8cc5b51998d819e712b59  2020-02-22 17:05:00 UTC   \n",
       "1224239  faf937a4a11c51e03223c045aeea0cd1464abc64  2020-02-21 18:55:00 UTC   \n",
       "\n",
       "            HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0               heat          auto               689                      693   \n",
       "1               heat          hold               700                      701   \n",
       "2               heat          hold               687                      748   \n",
       "3               heat          hold               702                      705   \n",
       "4        auxHeatOnly          hold               697                      695   \n",
       "...              ...           ...               ...                      ...   \n",
       "1224234         auto          auto               691                      760   \n",
       "1224235         auto          hold               686                      760   \n",
       "1224236         auto          auto               692                      760   \n",
       "1224238         auto          hold               696                      760   \n",
       "1224239         auto          auto               677                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            693            MA         Fitchburg    \n",
       "1                            701            MA    West Brookfield   \n",
       "2                            613            MA          Stoughton   \n",
       "3                            705            MA          Brookline   \n",
       "4                            695            MA  Washington Square   \n",
       "...                          ...           ...                ...   \n",
       "1224234                      690            MA             Boston   \n",
       "1224235                      690            MA      Yarmouth port   \n",
       "1224236                      690            MA             Boston   \n",
       "1224238                      700            MA      East Falmouth   \n",
       "1224239                      680            MA            Sudbury   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         80             False         False            False   \n",
       "1                          0             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         20             False         False             True   \n",
       "4                         45              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1224234                    0             False         False             True   \n",
       "1224235                    5             False         False            False   \n",
       "1224236                    0             False         False             True   \n",
       "1224238                    0             False         False            False   \n",
       "1224239                   57              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1224234                  Electric  \n",
       "1224235                       Gas  \n",
       "1224236                  Electric  \n",
       "1224238                       Gas  \n",
       "1224239                       Gas  \n",
       "\n",
       "[1106330 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/MA/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/MA-day/2021-feb-day-MA.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2b36525fa52e0f95d133cf51620e7ccaa032f9dd</td>\n",
       "      <td>2021-02-05 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>767</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47095ff07402f024224b8ff145f939ad82b5bd72</td>\n",
       "      <td>2021-02-06 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>727</td>\n",
       "      <td>727</td>\n",
       "      <td>MA</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c62f1d61ff1f925425c1492275d3045d961fd3b1</td>\n",
       "      <td>2021-02-06 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>617</td>\n",
       "      <td>613</td>\n",
       "      <td>613</td>\n",
       "      <td>MA</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e56212e2ddecb27c59eaa6464f08a05be98dc947</td>\n",
       "      <td>2021-02-27 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>MA</td>\n",
       "      <td>Belmont</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7d5d2ade30ab59fbb6c0ce2e4b4bd9b6b0c83300</td>\n",
       "      <td>2021-02-08 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>MA</td>\n",
       "      <td>Falmouth</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858697</th>\n",
       "      <td>66ef8c0b1d7397c741a5152bdf2f555e3cb9350e</td>\n",
       "      <td>2021-02-26 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858698</th>\n",
       "      <td>a013c8f83f25a67eabc0e9ee4b1ba76a2d9f5dbe</td>\n",
       "      <td>2021-02-08 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Somerville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858699</th>\n",
       "      <td>f765cc16f017612fca90aca7449e51aa1d0c8e98</td>\n",
       "      <td>2021-02-28 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858700</th>\n",
       "      <td>30d9edfacdb04d1733e366d889ff2ea25052e519</td>\n",
       "      <td>2021-02-16 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Tewksbury</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>858701</th>\n",
       "      <td>0d3dd4457fb8476c00d0319206d0e6c368e2bb37</td>\n",
       "      <td>2021-02-03 10:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Danvers</td>\n",
       "      <td>37</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>809302 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2b36525fa52e0f95d133cf51620e7ccaa032f9dd  2021-02-05 13:20:00 UTC   \n",
       "1       47095ff07402f024224b8ff145f939ad82b5bd72  2021-02-06 16:50:00 UTC   \n",
       "2       c62f1d61ff1f925425c1492275d3045d961fd3b1  2021-02-06 16:30:00 UTC   \n",
       "3       e56212e2ddecb27c59eaa6464f08a05be98dc947  2021-02-27 14:35:00 UTC   \n",
       "4       7d5d2ade30ab59fbb6c0ce2e4b4bd9b6b0c83300  2021-02-08 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "858697  66ef8c0b1d7397c741a5152bdf2f555e3cb9350e  2021-02-26 15:15:00 UTC   \n",
       "858698  a013c8f83f25a67eabc0e9ee4b1ba76a2d9f5dbe  2021-02-08 15:10:00 UTC   \n",
       "858699  f765cc16f017612fca90aca7449e51aa1d0c8e98  2021-02-28 18:05:00 UTC   \n",
       "858700  30d9edfacdb04d1733e366d889ff2ea25052e519  2021-02-16 14:50:00 UTC   \n",
       "858701  0d3dd4457fb8476c00d0319206d0e6c368e2bb37  2021-02-03 10:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               692                      767   \n",
       "1          heat          hold               726                      727   \n",
       "2          heat          hold               617                      613   \n",
       "3          heat          hold               683                      687   \n",
       "4          heat          hold               712                      716   \n",
       "...         ...           ...               ...                      ...   \n",
       "858697     auto          hold               704                      760   \n",
       "858698     auto          hold               707                      760   \n",
       "858699     auto          hold               705                      760   \n",
       "858700     auto          hold               694                      760   \n",
       "858701     auto          hold               689                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           698            MA       Newton   \n",
       "1                           727            MA  Springfield   \n",
       "2                           613            MA       Woburn   \n",
       "3                           687            MA      Belmont   \n",
       "4                           716            MA     Falmouth   \n",
       "...                         ...           ...          ...   \n",
       "858697                      690            MA       Boston   \n",
       "858698                      710            MA   Somerville   \n",
       "858699                      680            MA       Boston   \n",
       "858700                      700            MA    Tewksbury   \n",
       "858701                      700            MA      Danvers   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                        30             False         False            False   \n",
       "2                        57             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "858697                    0             False         False             True   \n",
       "858698                    0             False         False            False   \n",
       "858699                    0             False         False             True   \n",
       "858700                   40             False         False            False   \n",
       "858701                   37              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "858697                  Electric  \n",
       "858698                       Gas  \n",
       "858699                  Electric  \n",
       "858700                       Gas  \n",
       "858701                  Electric  \n",
       "\n",
       "[809302 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/MA/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>682.666667</td>\n",
       "      <td>659.380282</td>\n",
       "      <td>659.333333</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>696.840000</td>\n",
       "      <td>697.709091</td>\n",
       "      <td>697.242424</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Boston</td>\n",
       "      <td>680.270270</td>\n",
       "      <td>765.945946</td>\n",
       "      <td>688.378378</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boston</td>\n",
       "      <td>679.235294</td>\n",
       "      <td>793.529412</td>\n",
       "      <td>675.294118</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0373b135029a70f677ca2ad36469b0088d3c56bf</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>688.181818</td>\n",
       "      <td>687.545455</td>\n",
       "      <td>685.909091</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westford</td>\n",
       "      <td>700.841270</td>\n",
       "      <td>707.873016</td>\n",
       "      <td>707.714286</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1163</th>\n",
       "      <td>ff19754ee5d7c1c54c2d4d54707dd93ef9d562af</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Reading</td>\n",
       "      <td>713.985294</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164</th>\n",
       "      <td>ff30499eeb6176cf4df942531d3cd4ef45a94114</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Northborough</td>\n",
       "      <td>717.333333</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1165</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>678.009662</td>\n",
       "      <td>689.318841</td>\n",
       "      <td>688.748792</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1166</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>699.504697</td>\n",
       "      <td>704.001708</td>\n",
       "      <td>703.856533</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7398 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     021bc49d2737cfc5291795252754c8c84d99fb90   feb  2017     heat   \n",
       "1     021bc49d2737cfc5291795252754c8c84d99fb90   feb  2017     heat   \n",
       "2     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   feb  2017     auto   \n",
       "3     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   feb  2017     auto   \n",
       "4     0373b135029a70f677ca2ad36469b0088d3c56bf   feb  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1162  feb0e1fc76a69b0d234d4874cf27bae767b20c81   feb  2021     heat   \n",
       "1163  ff19754ee5d7c1c54c2d4d54707dd93ef9d562af   feb  2021     heat   \n",
       "1164  ff30499eeb6176cf4df942531d3cd4ef45a94114   feb  2021     heat   \n",
       "1165  fff0787bec7315864a164645420fade19da92725   feb  2021     heat   \n",
       "1166  fffa05184379d65eec4deeb00b14374669d0ff1e   feb  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto    Plainville            682.666667   \n",
       "1             hold    Plainville            696.840000   \n",
       "2             auto        Boston            680.270270   \n",
       "3             hold        Boston            679.235294   \n",
       "4             hold     Hopkinton            688.181818   \n",
       "...            ...           ...                   ...   \n",
       "1162          hold      Westford            700.841270   \n",
       "1163          hold       Reading            713.985294   \n",
       "1164          hold  Northborough            717.333333   \n",
       "1165          hold        Agawam            678.009662   \n",
       "1166          hold         Salem            699.504697   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      659.380282                   659.333333   \n",
       "1                      697.709091                   697.242424   \n",
       "2                      765.945946                   688.378378   \n",
       "3                      793.529412                   675.294118   \n",
       "4                      687.545455                   685.909091   \n",
       "...                           ...                          ...   \n",
       "1162                   707.873016                   707.714286   \n",
       "1163                   700.000000                   700.000000   \n",
       "1164                   720.000000                   720.000000   \n",
       "1165                   689.318841                   688.748792   \n",
       "1166                   704.001708                   703.856533   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    35.0             False         False            False  \n",
       "1                    35.0             False         False            False  \n",
       "2                   100.0             False         False            False  \n",
       "3                   100.0             False         False            False  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1162                 30.0              True         False            False  \n",
       "1163                  0.0             False         False            False  \n",
       "1164                 25.0              True         False            False  \n",
       "1165                 95.0             False         False            False  \n",
       "1166                  6.0             False         False            False  \n",
       "\n",
       "[7398 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/feb/\" + file)\n",
    "    MA_feb = pd.concat([MA_feb, df])\n",
    "    \n",
    "MA_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_feb.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/MA-day/2017-jun-day-MA.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8ae27191443d0d351cb5ef7ee29a94b3f9e6fdce</td>\n",
       "      <td>2017-06-28 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>MA</td>\n",
       "      <td>Tewksbury</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d8ee6f6e67e0167eec5f533495bcac32e5ac4356</td>\n",
       "      <td>2017-06-19 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>Shrewsbury</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>64059fed844db9fa2fa539103ef8e04b9304b147</td>\n",
       "      <td>2017-06-15 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wilmington</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>191ca983b162d2d36523b3216c4f93113f33c402</td>\n",
       "      <td>2017-06-29 10:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MA</td>\n",
       "      <td>Belmont</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5216a74352761806cb0e7841c37ad453c621cfb6</td>\n",
       "      <td>2017-06-12 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365889</th>\n",
       "      <td>fb0c545f29e0af345be86c2a4d291f460ca205da</td>\n",
       "      <td>2017-06-08 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365892</th>\n",
       "      <td>fb0c545f29e0af345be86c2a4d291f460ca205da</td>\n",
       "      <td>2017-06-08 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365895</th>\n",
       "      <td>fb0c545f29e0af345be86c2a4d291f460ca205da</td>\n",
       "      <td>2017-06-12 10:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365897</th>\n",
       "      <td>fb0c545f29e0af345be86c2a4d291f460ca205da</td>\n",
       "      <td>2017-06-09 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365898</th>\n",
       "      <td>fb0c545f29e0af345be86c2a4d291f460ca205da</td>\n",
       "      <td>2017-06-13 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>650</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>336656 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8ae27191443d0d351cb5ef7ee29a94b3f9e6fdce  2017-06-28 17:15:00 UTC   \n",
       "1       d8ee6f6e67e0167eec5f533495bcac32e5ac4356  2017-06-19 17:35:00 UTC   \n",
       "2       64059fed844db9fa2fa539103ef8e04b9304b147  2017-06-15 18:40:00 UTC   \n",
       "3       191ca983b162d2d36523b3216c4f93113f33c402  2017-06-29 10:35:00 UTC   \n",
       "4       5216a74352761806cb0e7841c37ad453c621cfb6  2017-06-12 14:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "365889  fb0c545f29e0af345be86c2a4d291f460ca205da  2017-06-08 18:10:00 UTC   \n",
       "365892  fb0c545f29e0af345be86c2a4d291f460ca205da  2017-06-08 15:25:00 UTC   \n",
       "365895  fb0c545f29e0af345be86c2a4d291f460ca205da  2017-06-12 10:55:00 UTC   \n",
       "365897  fb0c545f29e0af345be86c2a4d291f460ca205da  2017-06-09 18:00:00 UTC   \n",
       "365898  fb0c545f29e0af345be86c2a4d291f460ca205da  2017-06-13 12:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               699                      695   \n",
       "1          cool          hold               757                      740   \n",
       "2          cool          auto               701                      700   \n",
       "3          cool          hold               720                      750   \n",
       "4          cool          hold               745                      760   \n",
       "...         ...           ...               ...                      ...   \n",
       "365889     heat          hold               709                      650   \n",
       "365892     heat          hold               685                      650   \n",
       "365895     heat          hold               689                      650   \n",
       "365897     heat          hold               691                      650   \n",
       "365898     heat          hold               699                      650   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           645            MA      Tewksbury   \n",
       "1                           740            MA     Shrewsbury   \n",
       "2                           630            MA     Wilmington   \n",
       "3                           750            MA        Belmont   \n",
       "4                           760            MA      Westfield   \n",
       "...                         ...           ...            ...   \n",
       "365889                      620            MA  North Andover   \n",
       "365892                      620            MA  North Andover   \n",
       "365895                      620            MA  North Andover   \n",
       "365897                      620            MA  North Andover   \n",
       "365898                      620            MA  North Andover   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        16             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "365889                    0             False         False            False   \n",
       "365892                    0             False         False            False   \n",
       "365895                    0             False         False            False   \n",
       "365897                    0             False         False            False   \n",
       "365898                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "365889                       Gas  \n",
       "365892                       Gas  \n",
       "365895                       Gas  \n",
       "365897                       Gas  \n",
       "365898                       Gas  \n",
       "\n",
       "[336656 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/MA/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/MA-day/2018-jun-day-MA.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24835d6253ca2ccba6d6d7d526a1d89284a19395</td>\n",
       "      <td>2018-06-07 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>MA</td>\n",
       "      <td>Norwell</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1497fc6df99e8c408e6074a08989b17cc706411d</td>\n",
       "      <td>2018-06-15 11:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>MA</td>\n",
       "      <td>Swansea</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9c8b53bdd50859c24aab6bbbca68bc7f6db011b3</td>\n",
       "      <td>2018-06-22 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Scituate</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d6800c4898013f082ef2781dfd50b7284a23eae6</td>\n",
       "      <td>2018-06-19 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ludlow</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>de128735aced8423c9f0d9cdca65c608d1b9814a</td>\n",
       "      <td>2018-06-02 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>Methuen</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755228</th>\n",
       "      <td>dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15</td>\n",
       "      <td>2018-06-05 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Billerica</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755237</th>\n",
       "      <td>dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15</td>\n",
       "      <td>2018-06-02 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Billerica</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755245</th>\n",
       "      <td>dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15</td>\n",
       "      <td>2018-06-16 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Billerica</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755247</th>\n",
       "      <td>dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15</td>\n",
       "      <td>2018-06-21 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Billerica</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755249</th>\n",
       "      <td>dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15</td>\n",
       "      <td>2018-06-24 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>North Billerica</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>660986 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       24835d6253ca2ccba6d6d7d526a1d89284a19395  2018-06-07 13:10:00 UTC   \n",
       "1       1497fc6df99e8c408e6074a08989b17cc706411d  2018-06-15 11:55:00 UTC   \n",
       "2       9c8b53bdd50859c24aab6bbbca68bc7f6db011b3  2018-06-22 14:40:00 UTC   \n",
       "4       d6800c4898013f082ef2781dfd50b7284a23eae6  2018-06-19 19:25:00 UTC   \n",
       "5       de128735aced8423c9f0d9cdca65c608d1b9814a  2018-06-02 18:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "755228  dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15  2018-06-05 12:55:00 UTC   \n",
       "755237  dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15  2018-06-02 13:30:00 UTC   \n",
       "755245  dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15  2018-06-16 15:05:00 UTC   \n",
       "755247  dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15  2018-06-21 13:15:00 UTC   \n",
       "755249  dffd7ac8e3855165bc4b4fa3be26ecdc9c7f5f15  2018-06-24 17:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               730                      730   \n",
       "1          auto          hold               731                      740   \n",
       "2          cool          hold               716                      710   \n",
       "4          cool          hold               748                      750   \n",
       "5          cool          auto               748                      760   \n",
       "...         ...           ...               ...                      ...   \n",
       "755228     cool          hold               676                      750   \n",
       "755237     cool          hold               720                      720   \n",
       "755245     cool          hold               710                      740   \n",
       "755247     cool          auto               703                      760   \n",
       "755249     cool          hold               721                      740   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                           730            MA          Norwell   \n",
       "1                           660            MA          Swansea   \n",
       "2                           710            MA         Scituate   \n",
       "4                           750            MA           Ludlow   \n",
       "5                           720            MA          Methuen   \n",
       "...                         ...           ...              ...   \n",
       "755228                      750            MA  North Billerica   \n",
       "755237                      720            MA  North Billerica   \n",
       "755245                      740            MA  North Billerica   \n",
       "755247                      620            MA  North Billerica   \n",
       "755249                      740            MA  North Billerica   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        57             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "2                        35             False         False            False   \n",
       "4                        55             False         False            False   \n",
       "5                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "755228                   17             False         False            False   \n",
       "755237                   17             False         False            False   \n",
       "755245                   17             False         False            False   \n",
       "755247                   17             False         False            False   \n",
       "755249                   17             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "755228                       Gas  \n",
       "755237                       Gas  \n",
       "755245                       Gas  \n",
       "755247                       Gas  \n",
       "755249                       Gas  \n",
       "\n",
       "[660986 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/MA/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/MA-day/2019-jun-day-MA.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f9b6014fd057d1c4c38f3997938d9ce2dab5c524</td>\n",
       "      <td>2019-06-29 11:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>MA</td>\n",
       "      <td>Malden</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3663f1ce2832866260887dec4444175f80a858d1</td>\n",
       "      <td>2019-06-02 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>784</td>\n",
       "      <td>602</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stoughton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2019-06-14 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>718</td>\n",
       "      <td>648</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ba5e6315d62315cea9525ba33c92723eca1400f2</td>\n",
       "      <td>2019-06-09 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>780</td>\n",
       "      <td>747</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>dc689991641ec6c0367705a72c31c5c8dc57a55b</td>\n",
       "      <td>2019-06-23 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162273</th>\n",
       "      <td>29d336fb504289e524367a74a833498f7d3117ee</td>\n",
       "      <td>2019-06-10 11:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162274</th>\n",
       "      <td>2c5ccdd34fdea9cb4872556d185ed9420d376bca</td>\n",
       "      <td>2019-06-08 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Millis</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162275</th>\n",
       "      <td>b0a85c638a961bf0f52136290a5b9e864c864fef</td>\n",
       "      <td>2019-06-30 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Walpole</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162276</th>\n",
       "      <td>2f9aad858bca592e12712ec270293605972fe162</td>\n",
       "      <td>2019-06-25 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1162277</th>\n",
       "      <td>7aa9f18d5edb786630941bdcfbc227935b4e39f6</td>\n",
       "      <td>2019-06-07 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>749</td>\n",
       "      <td>800</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Acton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1010740 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        f9b6014fd057d1c4c38f3997938d9ce2dab5c524  2019-06-29 11:20:00 UTC   \n",
       "2        3663f1ce2832866260887dec4444175f80a858d1  2019-06-02 17:40:00 UTC   \n",
       "3        675e97e14602b3adfee227ef565c0205a1f0d40c  2019-06-14 14:50:00 UTC   \n",
       "5        ba5e6315d62315cea9525ba33c92723eca1400f2  2019-06-09 14:50:00 UTC   \n",
       "6        dc689991641ec6c0367705a72c31c5c8dc57a55b  2019-06-23 13:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1162273  29d336fb504289e524367a74a833498f7d3117ee  2019-06-10 11:20:00 UTC   \n",
       "1162274  2c5ccdd34fdea9cb4872556d185ed9420d376bca  2019-06-08 13:30:00 UTC   \n",
       "1162275  b0a85c638a961bf0f52136290a5b9e864c864fef  2019-06-30 17:50:00 UTC   \n",
       "1162276  2f9aad858bca592e12712ec270293605972fe162  2019-06-25 15:05:00 UTC   \n",
       "1162277  7aa9f18d5edb786630941bdcfbc227935b4e39f6  2019-06-07 17:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               739                      734   \n",
       "2           auto          hold               670                      784   \n",
       "3           auto          hold               692                      718   \n",
       "5           cool          hold               732                      780   \n",
       "6           cool          hold               708                      765   \n",
       "...          ...           ...               ...                      ...   \n",
       "1162273     cool          hold               729                      760   \n",
       "1162274     cool          hold               747                      760   \n",
       "1162275     cool          hold               752                      760   \n",
       "1162276     cool          auto               743                      760   \n",
       "1162277     cool          auto               749                      800   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            734            MA            Malden   \n",
       "2                            602            MA         Stoughton   \n",
       "3                            648            MA     Yarmouth port   \n",
       "5                            747            MA  West Springfield   \n",
       "6                            765            MA            Natick   \n",
       "...                          ...           ...               ...   \n",
       "1162273                      760            MA            Natick   \n",
       "1162274                      760            MA            Millis   \n",
       "1162275                      760            MA           Walpole   \n",
       "1162276                      760            MA           Andover   \n",
       "1162277                      760            MA             Acton   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "5                          0              True         False            False   \n",
       "6                        117             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1162273                  120             False         False            False   \n",
       "1162274                   20             False         False            False   \n",
       "1162275                   10             False         False            False   \n",
       "1162276                    0             False         False            False   \n",
       "1162277                    5             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "5                             Gas  \n",
       "6                             Gas  \n",
       "...                           ...  \n",
       "1162273                       Gas  \n",
       "1162274                       Gas  \n",
       "1162275                       Gas  \n",
       "1162276                       Gas  \n",
       "1162277                       Gas  \n",
       "\n",
       "[1010740 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/MA/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/MA-day/2020-jun-day-MA.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0e76511b11c6f38465ad968c99410de1dae1682b</td>\n",
       "      <td>2020-06-09 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7ff3d7cca7faf57f8c0db77afc2384831dc72198</td>\n",
       "      <td>2020-06-25 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>732</td>\n",
       "      <td>732</td>\n",
       "      <td>MA</td>\n",
       "      <td>Dracut</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e918c66c39ef6c889f22417469fd5a9091b254de</td>\n",
       "      <td>2020-06-17 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>800</td>\n",
       "      <td>755</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>94173d930de3af68d64b8c362b6653f743bd3034</td>\n",
       "      <td>2020-06-12 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>MA</td>\n",
       "      <td>Waltham</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>42aa7bb2dded717b64bc8d905b7c91cf69ad76ce</td>\n",
       "      <td>2020-06-30 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>723</td>\n",
       "      <td>723</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188744</th>\n",
       "      <td>0eec86e7c8a32910d0f874e681549f0116a52dbc</td>\n",
       "      <td>2020-06-14 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>800</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188745</th>\n",
       "      <td>d09b257581854b09bbab6268b6a323b9330f412c</td>\n",
       "      <td>2020-06-29 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Millis</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188746</th>\n",
       "      <td>cd4c87b39e30363018921df9f70f38310673248f</td>\n",
       "      <td>2020-06-16 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188747</th>\n",
       "      <td>776c0f62a9e023a8ba11ae436e47b5bcec4db5c2</td>\n",
       "      <td>2020-06-23 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Leominster</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1188748</th>\n",
       "      <td>d9bab4204472e1946af036043f48784bc9a5c99f</td>\n",
       "      <td>2020-06-10 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Marlborough</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1033614 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        0e76511b11c6f38465ad968c99410de1dae1682b  2020-06-09 13:20:00 UTC   \n",
       "3        7ff3d7cca7faf57f8c0db77afc2384831dc72198  2020-06-25 18:35:00 UTC   \n",
       "4        e918c66c39ef6c889f22417469fd5a9091b254de  2020-06-17 13:20:00 UTC   \n",
       "5        94173d930de3af68d64b8c362b6653f743bd3034  2020-06-12 18:25:00 UTC   \n",
       "6        42aa7bb2dded717b64bc8d905b7c91cf69ad76ce  2020-06-30 11:50:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1188744  0eec86e7c8a32910d0f874e681549f0116a52dbc  2020-06-14 16:10:00 UTC   \n",
       "1188745  d09b257581854b09bbab6268b6a323b9330f412c  2020-06-29 16:50:00 UTC   \n",
       "1188746  cd4c87b39e30363018921df9f70f38310673248f  2020-06-16 18:20:00 UTC   \n",
       "1188747  776c0f62a9e023a8ba11ae436e47b5bcec4db5c2  2020-06-23 17:50:00 UTC   \n",
       "1188748  d9bab4204472e1946af036043f48784bc9a5c99f  2020-06-10 19:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           heat          hold               730                      728   \n",
       "3           cool          hold               736                      732   \n",
       "4           cool          hold               757                      800   \n",
       "5           cool          hold               712                      689   \n",
       "6           cool          hold               722                      723   \n",
       "...          ...           ...               ...                      ...   \n",
       "1188744     cool          auto               708                      800   \n",
       "1188745     cool          auto               746                      760   \n",
       "1188746     cool          hold               762                      760   \n",
       "1188747     cool          hold               770                      760   \n",
       "1188748     cool          hold               738                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "1                            728            MA      Andover   \n",
       "3                            732            MA       Dracut   \n",
       "4                            755            MA       Boston   \n",
       "5                            689            MA      Waltham   \n",
       "6                            723            MA       Boston   \n",
       "...                          ...           ...          ...   \n",
       "1188744                      760            MA   Plainville   \n",
       "1188745                      760            MA       Millis   \n",
       "1188746                      760            MA    Watertown   \n",
       "1188747                      760            MA   Leominster   \n",
       "1188748                      760            MA  Marlborough   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                          0              True         False            False   \n",
       "3                         20             False         False            False   \n",
       "4                         90             False         False            False   \n",
       "5                         67             False         False            False   \n",
       "6                         15             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1188744                    0             False         False            False   \n",
       "1188745                    0             False         False            False   \n",
       "1188746                    0             False         False            False   \n",
       "1188747                   10             False         False            False   \n",
       "1188748                   50              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "6                             Gas  \n",
       "...                           ...  \n",
       "1188744                       Gas  \n",
       "1188745                       Gas  \n",
       "1188746                       Gas  \n",
       "1188747                       Gas  \n",
       "1188748                       Gas  \n",
       "\n",
       "[1033614 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/MA/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/MA-day/2021-jun-day-MA.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d86ee209aa87d21a8b3f498f76848feb064038fb</td>\n",
       "      <td>2021-06-24 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>MA</td>\n",
       "      <td>Abington</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b0a85c638a961bf0f52136290a5b9e864c864fef</td>\n",
       "      <td>2021-06-07 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>MA</td>\n",
       "      <td>Walpole</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a40bf8df5fb9d5879f3e666835f4aaff110659e1</td>\n",
       "      <td>2021-06-12 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sudbury</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>47095ff07402f024224b8ff145f939ad82b5bd72</td>\n",
       "      <td>2021-06-11 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>MA</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6cce63f61dc10d4b8dbc47b4476e9c63372b662b</td>\n",
       "      <td>2021-06-27 10:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>662</td>\n",
       "      <td>MA</td>\n",
       "      <td>Needham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783300</th>\n",
       "      <td>713b0c0b4f4a8ceaff93d24b7c1153bce073ad19</td>\n",
       "      <td>2021-06-23 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Billerica</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783301</th>\n",
       "      <td>55126d4d600a68aee3e6477313d7445d9116eb4d</td>\n",
       "      <td>2021-06-28 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783302</th>\n",
       "      <td>90d0e4d3b202fea6ef8d17b9bd31595572c148d9</td>\n",
       "      <td>2021-06-09 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sharon</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783303</th>\n",
       "      <td>2c5ccdd34fdea9cb4872556d185ed9420d376bca</td>\n",
       "      <td>2021-06-28 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Millis</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783304</th>\n",
       "      <td>ce1e81833b30f4fd9e320f6698abff1195f71182</td>\n",
       "      <td>2021-06-16 10:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Billerica</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>712437 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d86ee209aa87d21a8b3f498f76848feb064038fb  2021-06-24 17:40:00 UTC   \n",
       "1       b0a85c638a961bf0f52136290a5b9e864c864fef  2021-06-07 17:55:00 UTC   \n",
       "3       a40bf8df5fb9d5879f3e666835f4aaff110659e1  2021-06-12 19:20:00 UTC   \n",
       "4       47095ff07402f024224b8ff145f939ad82b5bd72  2021-06-11 19:00:00 UTC   \n",
       "5       6cce63f61dc10d4b8dbc47b4476e9c63372b662b  2021-06-27 10:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "783300  713b0c0b4f4a8ceaff93d24b7c1153bce073ad19  2021-06-23 19:05:00 UTC   \n",
       "783301  55126d4d600a68aee3e6477313d7445d9116eb4d  2021-06-28 16:50:00 UTC   \n",
       "783302  90d0e4d3b202fea6ef8d17b9bd31595572c148d9  2021-06-09 19:45:00 UTC   \n",
       "783303  2c5ccdd34fdea9cb4872556d185ed9420d376bca  2021-06-28 14:45:00 UTC   \n",
       "783304  ce1e81833b30f4fd9e320f6698abff1195f71182  2021-06-16 10:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               769                      671   \n",
       "1          cool          hold               720                      729   \n",
       "3          cool          hold               717                      735   \n",
       "4          cool          hold               710                      707   \n",
       "5          auto          hold               712                      712   \n",
       "...         ...           ...               ...                      ...   \n",
       "783300     cool          hold               766                      760   \n",
       "783301     cool          hold               778                      760   \n",
       "783302     cool          hold               766                      760   \n",
       "783303     cool          hold               765                      760   \n",
       "783304     cool          hold               737                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           671            MA     Abington   \n",
       "1                           729            MA      Walpole   \n",
       "3                           685            MA      Sudbury   \n",
       "4                           707            MA  Springfield   \n",
       "5                           662            MA      Needham   \n",
       "...                         ...           ...          ...   \n",
       "783300                      760            MA    Billerica   \n",
       "783301                      760            MA    Cambridge   \n",
       "783302                      760            MA       Sharon   \n",
       "783303                      760            MA       Millis   \n",
       "783304                      760            MA    Billerica   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "3                        17             False         False            False   \n",
       "4                        30             False         False            False   \n",
       "5                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "783300                   25              True         False            False   \n",
       "783301                    0             False         False            False   \n",
       "783302                   30             False         False            False   \n",
       "783303                   20             False         False            False   \n",
       "783304                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "783300                       Gas  \n",
       "783301                       Gas  \n",
       "783302                       Gas  \n",
       "783303                       Gas  \n",
       "783304                       Gas  \n",
       "\n",
       "[712437 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/MA/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>697.923913</td>\n",
       "      <td>699.945652</td>\n",
       "      <td>699.945652</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>713.794872</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02382555cf0333ff1ad92861205814fd74616f88</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Hadley</td>\n",
       "      <td>807.142857</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02b696f3784a0b13fc9de7883fb76bd2c1d25fa5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Natick</td>\n",
       "      <td>716.000000</td>\n",
       "      <td>706.500000</td>\n",
       "      <td>698.000000</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Boston</td>\n",
       "      <td>751.541667</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1263</th>\n",
       "      <td>fe2bdeb6fe92670a4befad74d25a7061473e51f8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwood</td>\n",
       "      <td>736.869565</td>\n",
       "      <td>733.000000</td>\n",
       "      <td>683.000000</td>\n",
       "      <td>67.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1264</th>\n",
       "      <td>fe3de6f77ca9ec90aab29b2a5918552e6ff9229e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westborough</td>\n",
       "      <td>737.575436</td>\n",
       "      <td>734.186145</td>\n",
       "      <td>734.186145</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1265</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westford</td>\n",
       "      <td>698.208333</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1266</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>762.393939</td>\n",
       "      <td>739.090909</td>\n",
       "      <td>739.090909</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1267</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>713.350746</td>\n",
       "      <td>712.985075</td>\n",
       "      <td>712.985075</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7785 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     021bc49d2737cfc5291795252754c8c84d99fb90   jun  2017     heat   \n",
       "1     021bc49d2737cfc5291795252754c8c84d99fb90   jun  2017     heat   \n",
       "2     02382555cf0333ff1ad92861205814fd74616f88   jun  2017     cool   \n",
       "3     02b696f3784a0b13fc9de7883fb76bd2c1d25fa5   jun  2017     cool   \n",
       "4     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   jun  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1263  fe2bdeb6fe92670a4befad74d25a7061473e51f8   jun  2021     auto   \n",
       "1264  fe3de6f77ca9ec90aab29b2a5918552e6ff9229e   jun  2021     cool   \n",
       "1265  feb0e1fc76a69b0d234d4874cf27bae767b20c81   jun  2021     cool   \n",
       "1266  fff0787bec7315864a164645420fade19da92725   jun  2021     cool   \n",
       "1267  fffa05184379d65eec4deeb00b14374669d0ff1e   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto    Plainville            697.923913   \n",
       "1             hold    Plainville            713.794872   \n",
       "2             hold  South Hadley            807.142857   \n",
       "3             hold        Natick            716.000000   \n",
       "4             auto        Boston            751.541667   \n",
       "...            ...           ...                   ...   \n",
       "1263          hold       Norwood            736.869565   \n",
       "1264          hold   Westborough            737.575436   \n",
       "1265          hold      Westford            698.208333   \n",
       "1266          hold        Agawam            762.393939   \n",
       "1267          hold         Salem            713.350746   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      699.945652                   699.945652   \n",
       "1                      710.000000                   710.000000   \n",
       "2                      760.000000                   680.000000   \n",
       "3                      706.500000                   698.000000   \n",
       "4                      750.000000                   640.000000   \n",
       "...                           ...                          ...   \n",
       "1263                   733.000000                   683.000000   \n",
       "1264                   734.186145                   734.186145   \n",
       "1265                   695.000000                   695.000000   \n",
       "1266                   739.090909                   739.090909   \n",
       "1267                   712.985075                   712.985075   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    35.0             False         False            False  \n",
       "1                    35.0             False         False            False  \n",
       "2                    55.0             False         False            False  \n",
       "3                    15.0             False         False            False  \n",
       "4                   100.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1263                 67.0             False         False            False  \n",
       "1264                 20.0             False         False            False  \n",
       "1265                 30.0              True         False            False  \n",
       "1266                 95.0             False         False            False  \n",
       "1267                  6.0             False         False            False  \n",
       "\n",
       "[7785 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/jun/\" + file)\n",
    "    MA_jun = pd.concat([MA_jun, df])\n",
    "    \n",
    "MA_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_jun.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/MA-day/2017-jul-day-MA.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4adf3e8e83838554f299ffe2f2fb6df6d61df124</td>\n",
       "      <td>2017-07-03 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>andover</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10beb6030317184450b4cc3e50aa60fa142c3338</td>\n",
       "      <td>2017-07-22 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>MA</td>\n",
       "      <td>Bolton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16bb1e62b30edcaaa53e0a9c999e0851b4366ecf</td>\n",
       "      <td>2017-07-05 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>737</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>Woburn</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>480f2b2bf74550d02c2322a1c662f4b619c36b66</td>\n",
       "      <td>2017-07-02 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Ayer</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4adf3e8e83838554f299ffe2f2fb6df6d61df124</td>\n",
       "      <td>2017-07-17 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>andover</td>\n",
       "      <td>56</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420690</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-07-03 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>807</td>\n",
       "      <td>810</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420691</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-07-22 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>809</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420692</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-07-22 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>804</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420693</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-07-04 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>788</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420694</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-07-22 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>801</td>\n",
       "      <td>820</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>383423 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       4adf3e8e83838554f299ffe2f2fb6df6d61df124  2017-07-03 14:00:00 UTC   \n",
       "1       10beb6030317184450b4cc3e50aa60fa142c3338  2017-07-22 14:20:00 UTC   \n",
       "2       16bb1e62b30edcaaa53e0a9c999e0851b4366ecf  2017-07-05 18:45:00 UTC   \n",
       "3       480f2b2bf74550d02c2322a1c662f4b619c36b66  2017-07-02 17:55:00 UTC   \n",
       "4       4adf3e8e83838554f299ffe2f2fb6df6d61df124  2017-07-17 18:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "420690  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-07-03 18:00:00 UTC   \n",
       "420691  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-07-22 13:55:00 UTC   \n",
       "420692  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-07-22 12:50:00 UTC   \n",
       "420693  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-07-04 17:20:00 UTC   \n",
       "420694  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-07-22 11:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      740   \n",
       "1          cool          hold               749                      790   \n",
       "2          cool          auto               726                      737   \n",
       "3          auto          auto               745                      750   \n",
       "4          cool          hold               738                      740   \n",
       "...         ...           ...               ...                      ...   \n",
       "420690     cool          hold               807                      810   \n",
       "420691     cool          hold               809                      820   \n",
       "420692     cool          hold               804                      820   \n",
       "420693     cool          hold               788                      790   \n",
       "420694     cool          hold               801                      820   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState     City  Age_of_Home__years_  \\\n",
       "0                           740            MA  andover                   56   \n",
       "1                           790            MA   Bolton                    0   \n",
       "2                           740            MA   Woburn                    0   \n",
       "3                           700            MA     Ayer                    5   \n",
       "4                           740            MA  andover                   56   \n",
       "...                         ...           ...      ...                  ...   \n",
       "420690                      760            MA     Lynn                  120   \n",
       "420691                      760            MA     Lynn                  120   \n",
       "420692                      760            MA     Lynn                  120   \n",
       "420693                      760            MA     Lynn                  120   \n",
       "420694                      760            MA     Lynn                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "420690             False         False            False   \n",
       "420691             False         False            False   \n",
       "420692             False         False            False   \n",
       "420693             False         False            False   \n",
       "420694             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "420690                       Gas  \n",
       "420691                       Gas  \n",
       "420692                       Gas  \n",
       "420693                       Gas  \n",
       "420694                       Gas  \n",
       "\n",
       "[383423 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/MA/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/MA-day/2018-jul-day-MA.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c7aa3301a11d033d2f06fe9ad1632272dfce8e42</td>\n",
       "      <td>2018-07-28 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>772</td>\n",
       "      <td>840</td>\n",
       "      <td>840</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cebb4dfa590044a66b856b97bef9728c1f7c1f6e</td>\n",
       "      <td>2018-07-18 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>789</td>\n",
       "      <td>800</td>\n",
       "      <td>610</td>\n",
       "      <td>MA</td>\n",
       "      <td>Brookline</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c3127b6426b8d8914e6f9a869e243f3d206b228</td>\n",
       "      <td>2018-07-22 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MA</td>\n",
       "      <td>Merrimac</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0ec218045f34ba636a652384c8c0083f0c461e88</td>\n",
       "      <td>2018-07-21 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>MA</td>\n",
       "      <td>Billerica</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0ec218045f34ba636a652384c8c0083f0c461e88</td>\n",
       "      <td>2018-07-20 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>MA</td>\n",
       "      <td>Billerica</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963287</th>\n",
       "      <td>fa4d6c93022e9e972ebc80cbe2ca30f1eef8779c</td>\n",
       "      <td>2018-07-31 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westborough</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963288</th>\n",
       "      <td>15a6565ac0e28793b55ec5bc2f271fa9ec28cc77</td>\n",
       "      <td>2018-07-19 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963289</th>\n",
       "      <td>9885ebe9e6190eed190aea0b902d3331eb4d57d8</td>\n",
       "      <td>2018-07-17 12:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963290</th>\n",
       "      <td>a187f3658ba313f5e4a8015c00fe7cd6e061a8ca</td>\n",
       "      <td>2018-07-29 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Salem</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963291</th>\n",
       "      <td>9885ebe9e6190eed190aea0b902d3331eb4d57d8</td>\n",
       "      <td>2018-07-29 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>847319 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c7aa3301a11d033d2f06fe9ad1632272dfce8e42  2018-07-28 14:10:00 UTC   \n",
       "2       cebb4dfa590044a66b856b97bef9728c1f7c1f6e  2018-07-18 15:00:00 UTC   \n",
       "3       6c3127b6426b8d8914e6f9a869e243f3d206b228  2018-07-22 15:35:00 UTC   \n",
       "4       0ec218045f34ba636a652384c8c0083f0c461e88  2018-07-21 13:50:00 UTC   \n",
       "5       0ec218045f34ba636a652384c8c0083f0c461e88  2018-07-20 15:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "963287  fa4d6c93022e9e972ebc80cbe2ca30f1eef8779c  2018-07-31 11:35:00 UTC   \n",
       "963288  15a6565ac0e28793b55ec5bc2f271fa9ec28cc77  2018-07-19 13:30:00 UTC   \n",
       "963289  9885ebe9e6190eed190aea0b902d3331eb4d57d8  2018-07-17 12:20:00 UTC   \n",
       "963290  a187f3658ba313f5e4a8015c00fe7cd6e061a8ca  2018-07-29 17:55:00 UTC   \n",
       "963291  9885ebe9e6190eed190aea0b902d3331eb4d57d8  2018-07-29 16:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               772                      840   \n",
       "2          auto          hold               789                      800   \n",
       "3          auto          hold               682                      685   \n",
       "4          cool          hold               715                      820   \n",
       "5          cool          hold               736                      820   \n",
       "...         ...           ...               ...                      ...   \n",
       "963287     cool          hold               751                      760   \n",
       "963288     cool          hold               764                      760   \n",
       "963289     cool          hold               757                      760   \n",
       "963290     cool          hold               767                      760   \n",
       "963291     cool          auto               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           840            MA       Boston   \n",
       "2                           610            MA    Brookline   \n",
       "3                           635            MA     Merrimac   \n",
       "4                           820            MA    Billerica   \n",
       "5                           820            MA    Billerica   \n",
       "...                         ...           ...          ...   \n",
       "963287                      760            MA  Westborough   \n",
       "963288                      760            MA    Cambridge   \n",
       "963289                      760            MA       Boston   \n",
       "963290                      760            MA        Salem   \n",
       "963291                      760            MA       Boston   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20              True         False            False   \n",
       "2                        90             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        57             False         False            False   \n",
       "5                        57             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "963287                   20             False         False            False   \n",
       "963288                   90              True         False            False   \n",
       "963289                  120             False         False            False   \n",
       "963290                    0              True         False             True   \n",
       "963291                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "963287                       Gas  \n",
       "963288                       Gas  \n",
       "963289                       Gas  \n",
       "963290                  Electric  \n",
       "963291                       Gas  \n",
       "\n",
       "[847319 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/MA/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/MA-day/2019-jul-day-MA.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>95b6f6d10db6f24db56877600e5998598834ff08</td>\n",
       "      <td>2019-07-01 12:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>710</td>\n",
       "      <td>615</td>\n",
       "      <td>MA</td>\n",
       "      <td>Medford</td>\n",
       "      <td>115</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b35daa351e18ba1a8fd61de252be873eba95eb92</td>\n",
       "      <td>2019-07-05 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2019-07-01 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>718</td>\n",
       "      <td>638</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f2aa06513cc33dec00b5fa081b9d7a2599397335</td>\n",
       "      <td>2019-07-22 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>771</td>\n",
       "      <td>675</td>\n",
       "      <td>MA</td>\n",
       "      <td>Malden</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2b36525fa52e0f95d133cf51620e7ccaa032f9dd</td>\n",
       "      <td>2019-07-30 10:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>757</td>\n",
       "      <td>689</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381456</th>\n",
       "      <td>d77bb5a82db82e4b2e73fb92d56dfff41e9edb60</td>\n",
       "      <td>2019-07-24 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stow</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381457</th>\n",
       "      <td>d8cec133a6626dabde8d0e6e55c51a4a3dda7992</td>\n",
       "      <td>2019-07-17 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Orleans</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381458</th>\n",
       "      <td>b8d68e13c08db2464631bc28b81e8d751e6c0940</td>\n",
       "      <td>2019-07-21 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381459</th>\n",
       "      <td>a9f1dc7382d7ffebe1fc7d9b497da358d016b460</td>\n",
       "      <td>2019-07-28 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Quincy</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381460</th>\n",
       "      <td>b1e55c1b02cf26c35b6ecc3243196e6e9ded9eaf</td>\n",
       "      <td>2019-07-06 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newbury</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1230843 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        95b6f6d10db6f24db56877600e5998598834ff08  2019-07-01 12:10:00 UTC   \n",
       "1        b35daa351e18ba1a8fd61de252be873eba95eb92  2019-07-05 11:00:00 UTC   \n",
       "4        675e97e14602b3adfee227ef565c0205a1f0d40c  2019-07-01 17:25:00 UTC   \n",
       "6        f2aa06513cc33dec00b5fa081b9d7a2599397335  2019-07-22 19:55:00 UTC   \n",
       "7        2b36525fa52e0f95d133cf51620e7ccaa032f9dd  2019-07-30 10:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1381456  d77bb5a82db82e4b2e73fb92d56dfff41e9edb60  2019-07-24 15:05:00 UTC   \n",
       "1381457  d8cec133a6626dabde8d0e6e55c51a4a3dda7992  2019-07-17 12:05:00 UTC   \n",
       "1381458  b8d68e13c08db2464631bc28b81e8d751e6c0940  2019-07-21 17:50:00 UTC   \n",
       "1381459  a9f1dc7382d7ffebe1fc7d9b497da358d016b460  2019-07-28 10:40:00 UTC   \n",
       "1381460  b1e55c1b02cf26c35b6ecc3243196e6e9ded9eaf  2019-07-06 18:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               704                      710   \n",
       "1           cool          hold               727                      725   \n",
       "4           auto          hold               717                      718   \n",
       "6           cool          auto               783                      771   \n",
       "7           auto          hold               762                      757   \n",
       "...          ...           ...               ...                      ...   \n",
       "1381456     cool          hold               744                      760   \n",
       "1381457     cool          hold               757                      760   \n",
       "1381458     cool          auto               761                      760   \n",
       "1381459     cool          hold               765                      760   \n",
       "1381460     cool          hold               761                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            615            MA        Medford   \n",
       "1                            725            MA         Boston   \n",
       "4                            638            MA  Yarmouth port   \n",
       "6                            675            MA         Malden   \n",
       "7                            689            MA         Newton   \n",
       "...                          ...           ...            ...   \n",
       "1381456                      760            MA           Stow   \n",
       "1381457                      760            MA        Orleans   \n",
       "1381458                      760            MA         Newton   \n",
       "1381459                      760            MA         Quincy   \n",
       "1381460                      760            MA        Newbury   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        115             False         False            False   \n",
       "1                         95             False         False            False   \n",
       "4                          5             False         False            False   \n",
       "6                         60             False         False            False   \n",
       "7                          0              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1381456                   35             False         False            False   \n",
       "1381457                   50             False         False            False   \n",
       "1381458                   95             False         False            False   \n",
       "1381459                   30              True         False             True   \n",
       "1381460                    0             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "4                             Gas  \n",
       "6                             Gas  \n",
       "7                             Gas  \n",
       "...                           ...  \n",
       "1381456                       Gas  \n",
       "1381457                       Gas  \n",
       "1381458                       Gas  \n",
       "1381459                  Electric  \n",
       "1381460                  Electric  \n",
       "\n",
       "[1230843 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/MA/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/MA-day/2020-jul-day-MA.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6678829786741575133603ac56eb6e05c6ef69e7</td>\n",
       "      <td>2020-07-07 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sharon</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b35daa351e18ba1a8fd61de252be873eba95eb92</td>\n",
       "      <td>2020-07-06 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b9f8b3f0c5bf9aca91b52d35db733e9f3c89e775</td>\n",
       "      <td>2020-07-26 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8e9e331e3c90a487dd6829e421da385540a3394c</td>\n",
       "      <td>2020-07-09 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stoughton</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0e76511b11c6f38465ad968c99410de1dae1682b</td>\n",
       "      <td>2020-07-22 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>650</td>\n",
       "      <td>608</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349470</th>\n",
       "      <td>081c4bf06967787d4b0755732a5806483609785f</td>\n",
       "      <td>2020-07-12 12:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>800</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Holden</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349471</th>\n",
       "      <td>6d1364dc37770ca047de5dd1bda06cd9c516171a</td>\n",
       "      <td>2020-07-28 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Beverly</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349472</th>\n",
       "      <td>4ad3de70dff06f493c3ac1c8af9d8114c8ddf088</td>\n",
       "      <td>2020-07-21 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>osterville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349473</th>\n",
       "      <td>492784f1c9ee5262dfd9147baf61632e236f02db</td>\n",
       "      <td>2020-07-06 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Malden</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1349474</th>\n",
       "      <td>01c66208f7592bbe8a08a40fc76bde2cf4eebe66</td>\n",
       "      <td>2020-07-10 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1181954 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        6678829786741575133603ac56eb6e05c6ef69e7  2020-07-07 18:55:00 UTC   \n",
       "1        b35daa351e18ba1a8fd61de252be873eba95eb92  2020-07-06 13:45:00 UTC   \n",
       "2        b9f8b3f0c5bf9aca91b52d35db733e9f3c89e775  2020-07-26 13:50:00 UTC   \n",
       "3        8e9e331e3c90a487dd6829e421da385540a3394c  2020-07-09 14:30:00 UTC   \n",
       "4        0e76511b11c6f38465ad968c99410de1dae1682b  2020-07-22 12:50:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1349470  081c4bf06967787d4b0755732a5806483609785f  2020-07-12 12:55:00 UTC   \n",
       "1349471  6d1364dc37770ca047de5dd1bda06cd9c516171a  2020-07-28 14:15:00 UTC   \n",
       "1349472  4ad3de70dff06f493c3ac1c8af9d8114c8ddf088  2020-07-21 13:15:00 UTC   \n",
       "1349473  492784f1c9ee5262dfd9147baf61632e236f02db  2020-07-06 15:15:00 UTC   \n",
       "1349474  01c66208f7592bbe8a08a40fc76bde2cf4eebe66  2020-07-10 13:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               704                      699   \n",
       "1           auto          hold               731                      733   \n",
       "2           cool          hold               716                      716   \n",
       "3           auto          hold               720                      685   \n",
       "4           heat          hold               779                      650   \n",
       "...          ...           ...               ...                      ...   \n",
       "1349470     heat          hold               800                      760   \n",
       "1349471     cool          auto               747                      760   \n",
       "1349472     cool          hold               764                      760   \n",
       "1349473     cool          hold               770                      760   \n",
       "1349474     cool          hold               744                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                            699            MA      Sharon   \n",
       "1                            683            MA      Boston   \n",
       "2                            716            MA   Cambridge   \n",
       "3                            635            MA   Stoughton   \n",
       "4                            608            MA     Andover   \n",
       "...                          ...           ...         ...   \n",
       "1349470                      760            MA      Holden   \n",
       "1349471                      760            MA     Beverly   \n",
       "1349472                      760            MA  osterville   \n",
       "1349473                      760            MA      Malden   \n",
       "1349474                      760            MA      Natick   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         95             False         False            False   \n",
       "2                         25             False         False            False   \n",
       "3                          9             False         False            False   \n",
       "4                          0              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1349470                   67             False         False            False   \n",
       "1349471                    0             False         False            False   \n",
       "1349472                    0             False         False            False   \n",
       "1349473                    5              True         False            False   \n",
       "1349474                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1349470                       Gas  \n",
       "1349471                       Gas  \n",
       "1349472                       Gas  \n",
       "1349473                       Gas  \n",
       "1349474                       Gas  \n",
       "\n",
       "[1181954 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/MA/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/MA-day/2021-jul-day-MA.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b331ba3cac06cd9f10483bf601561561c6f0f494</td>\n",
       "      <td>2021-07-29 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Beverly</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2f7afe40aa596095509742aab44713dfae961c23</td>\n",
       "      <td>2021-07-08 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>667</td>\n",
       "      <td>MA</td>\n",
       "      <td>NORTHBOROUGH</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7b7278e067845496136350644936eb6905ca6d03</td>\n",
       "      <td>2021-07-18 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>MA</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6df6239ee8ec0c85b71826d3594886d038271ba0</td>\n",
       "      <td>2021-07-01 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1ca0e3157c543951006bbf7616cb647ad91bb7f0</td>\n",
       "      <td>2021-07-06 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>MA</td>\n",
       "      <td>East Longmeadow</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766591</th>\n",
       "      <td>0a5b934aac1e65d86d24f2ded2c420a5fe40cce9</td>\n",
       "      <td>2021-07-24 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766592</th>\n",
       "      <td>66d407ef8a3ca3965d3a011c4009793e2f7b6663</td>\n",
       "      <td>2021-07-01 10:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Framingham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766593</th>\n",
       "      <td>83d1cf93fa2f36bab79c5321d3cceccd291c6c80</td>\n",
       "      <td>2021-07-28 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Abington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766594</th>\n",
       "      <td>83d1cf93fa2f36bab79c5321d3cceccd291c6c80</td>\n",
       "      <td>2021-07-21 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Abington</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>766595</th>\n",
       "      <td>26bdc2e5c06bb6fc528e07bba516062023e73a6a</td>\n",
       "      <td>2021-07-25 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westford</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>692112 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b331ba3cac06cd9f10483bf601561561c6f0f494  2021-07-29 14:45:00 UTC   \n",
       "3       2f7afe40aa596095509742aab44713dfae961c23  2021-07-08 18:30:00 UTC   \n",
       "4       7b7278e067845496136350644936eb6905ca6d03  2021-07-18 19:25:00 UTC   \n",
       "5       6df6239ee8ec0c85b71826d3594886d038271ba0  2021-07-01 18:55:00 UTC   \n",
       "6       1ca0e3157c543951006bbf7616cb647ad91bb7f0  2021-07-06 11:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "766591  0a5b934aac1e65d86d24f2ded2c420a5fe40cce9  2021-07-24 11:00:00 UTC   \n",
       "766592  66d407ef8a3ca3965d3a011c4009793e2f7b6663  2021-07-01 10:10:00 UTC   \n",
       "766593  83d1cf93fa2f36bab79c5321d3cceccd291c6c80  2021-07-28 15:45:00 UTC   \n",
       "766594  83d1cf93fa2f36bab79c5321d3cceccd291c6c80  2021-07-21 11:00:00 UTC   \n",
       "766595  26bdc2e5c06bb6fc528e07bba516062023e73a6a  2021-07-25 18:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               706                      705   \n",
       "3          cool          hold               746                      750   \n",
       "4          auto          hold               702                      695   \n",
       "5          auto          hold               730                      735   \n",
       "6          cool          hold               666                      666   \n",
       "...         ...           ...               ...                      ...   \n",
       "766591     cool          hold               758                      760   \n",
       "766592     cool          hold               762                      760   \n",
       "766593     cool          hold               698                      760   \n",
       "766594     cool          hold               676                      760   \n",
       "766595     cool          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                           705            MA          Beverly   \n",
       "3                           667            MA     NORTHBOROUGH   \n",
       "4                           645            MA        Watertown   \n",
       "5                           685            MA           Boston   \n",
       "6                           666            MA  East Longmeadow   \n",
       "...                         ...           ...              ...   \n",
       "766591                      760            MA        Watertown   \n",
       "766592                      760            MA       Framingham   \n",
       "766593                      760            MA         Abington   \n",
       "766594                      760            MA         Abington   \n",
       "766595                      760            MA         Westford   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60              True         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "5                         0             False         False            False   \n",
       "6                        60              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "766591                   60             False         False            False   \n",
       "766592                    0             False         False            False   \n",
       "766593                    0             False         False            False   \n",
       "766594                    0             False         False            False   \n",
       "766595                   30              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "766591                       Gas  \n",
       "766592                       Gas  \n",
       "766593                       Gas  \n",
       "766594                       Gas  \n",
       "766595                       Gas  \n",
       "\n",
       "[692112 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/MA/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02382555cf0333ff1ad92861205814fd74616f88</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Hadley</td>\n",
       "      <td>789.254717</td>\n",
       "      <td>776.122642</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Boston</td>\n",
       "      <td>706.083333</td>\n",
       "      <td>705.083333</td>\n",
       "      <td>655.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boston</td>\n",
       "      <td>730.043664</td>\n",
       "      <td>731.244007</td>\n",
       "      <td>643.472603</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0373b135029a70f677ca2ad36469b0088d3c56bf</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>788.066667</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0373b135029a70f677ca2ad36469b0088d3c56bf</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>762.063830</td>\n",
       "      <td>783.944681</td>\n",
       "      <td>771.634043</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1003</th>\n",
       "      <td>fe3cbab0c676e65e325a9ce5c7f0d311cb9955f5</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>West Newton</td>\n",
       "      <td>768.133333</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>85.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1004</th>\n",
       "      <td>fe3de6f77ca9ec90aab29b2a5918552e6ff9229e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westborough</td>\n",
       "      <td>744.467602</td>\n",
       "      <td>741.478958</td>\n",
       "      <td>741.478958</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westford</td>\n",
       "      <td>732.545455</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1006</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>749.469565</td>\n",
       "      <td>739.343478</td>\n",
       "      <td>739.221739</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1007</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>719.226994</td>\n",
       "      <td>722.291411</td>\n",
       "      <td>722.260736</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7266 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     02382555cf0333ff1ad92861205814fd74616f88   jul  2017     cool   \n",
       "1     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   jul  2017     auto   \n",
       "2     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   jul  2017     auto   \n",
       "3     0373b135029a70f677ca2ad36469b0088d3c56bf   jul  2017     cool   \n",
       "4     0373b135029a70f677ca2ad36469b0088d3c56bf   jul  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1003  fe3cbab0c676e65e325a9ce5c7f0d311cb9955f5   jul  2021     cool   \n",
       "1004  fe3de6f77ca9ec90aab29b2a5918552e6ff9229e   jul  2021     cool   \n",
       "1005  feb0e1fc76a69b0d234d4874cf27bae767b20c81   jul  2021     cool   \n",
       "1006  fff0787bec7315864a164645420fade19da92725   jul  2021     cool   \n",
       "1007  fffa05184379d65eec4deeb00b14374669d0ff1e   jul  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold  South Hadley            789.254717   \n",
       "1             auto        Boston            706.083333   \n",
       "2             hold        Boston            730.043664   \n",
       "3             auto     Hopkinton            788.066667   \n",
       "4             hold     Hopkinton            762.063830   \n",
       "...            ...           ...                   ...   \n",
       "1003          hold   West Newton            768.133333   \n",
       "1004          hold   Westborough            744.467602   \n",
       "1005          hold      Westford            732.545455   \n",
       "1006          hold        Agawam            749.469565   \n",
       "1007          hold         Salem            719.226994   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      776.122642                   680.000000   \n",
       "1                      705.083333                   655.000000   \n",
       "2                      731.244007                   643.472603   \n",
       "3                      800.000000                   790.000000   \n",
       "4                      783.944681                   771.634043   \n",
       "...                           ...                          ...   \n",
       "1003                   780.000000                   760.000000   \n",
       "1004                   741.478958                   741.478958   \n",
       "1005                   730.000000                   730.000000   \n",
       "1006                   739.343478                   739.221739   \n",
       "1007                   722.291411                   722.260736   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    55.0             False         False            False  \n",
       "1                   100.0             False         False            False  \n",
       "2                   100.0             False         False            False  \n",
       "3                    30.0             False         False            False  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1003                 85.0             False         False            False  \n",
       "1004                 20.0             False         False            False  \n",
       "1005                 30.0              True         False            False  \n",
       "1006                 95.0             False         False            False  \n",
       "1007                  6.0             False         False            False  \n",
       "\n",
       "[7266 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/jul/\" + file)\n",
    "    MA_jul = pd.concat([MA_jul, df])\n",
    "    \n",
    "MA_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_jul.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/MA-day/2017-aug-day-MA.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2efbf9f49a144550c9821efe422dfccb14e06a22</td>\n",
       "      <td>2017-08-20 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>770</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Waltham</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ebc99093956c3a8d92c17b761d85030f6816d885</td>\n",
       "      <td>2017-08-25 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>710</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>Amesbury</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5266933ef1ff20f61e423d8c4e0fe75699edee45</td>\n",
       "      <td>2017-08-26 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wellesley</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8ad0d59172604246e2d07b9ef4847fb995abf693</td>\n",
       "      <td>2017-08-10 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3111820bb033721bc8af4374922be18bdf853c79</td>\n",
       "      <td>2017-08-12 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400690</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-08-01 11:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>830</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400691</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-08-09 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>756</td>\n",
       "      <td>748</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400692</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-08-25 10:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>840</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400693</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-08-14 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400694</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2017-08-19 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>805</td>\n",
       "      <td>800</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>360677 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2efbf9f49a144550c9821efe422dfccb14e06a22  2017-08-20 17:35:00 UTC   \n",
       "1       ebc99093956c3a8d92c17b761d85030f6816d885  2017-08-25 16:00:00 UTC   \n",
       "2       5266933ef1ff20f61e423d8c4e0fe75699edee45  2017-08-26 12:30:00 UTC   \n",
       "3       8ad0d59172604246e2d07b9ef4847fb995abf693  2017-08-10 16:25:00 UTC   \n",
       "4       3111820bb033721bc8af4374922be18bdf853c79  2017-08-12 16:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "400690  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-08-01 11:25:00 UTC   \n",
       "400691  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-08-09 18:45:00 UTC   \n",
       "400692  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-08-25 10:45:00 UTC   \n",
       "400693  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-08-14 16:40:00 UTC   \n",
       "400694  f72bd16584f430a000cd0586fbe83212553cb2d6  2017-08-19 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               767                      770   \n",
       "1          auto          hold               708                      710   \n",
       "2          cool          auto               696                      770   \n",
       "3          cool          hold               734                      730   \n",
       "4          cool          hold               726                      740   \n",
       "...         ...           ...               ...                      ...   \n",
       "400690     cool          hold               775                      830   \n",
       "400691     cool          hold               731                      756   \n",
       "400692     cool          hold               746                      840   \n",
       "400693     cool          hold               758                      760   \n",
       "400694     cool          hold               805                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           680            MA    Waltham                   27   \n",
       "1                           620            MA   Amesbury                   10   \n",
       "2                           700            MA  Wellesley                  105   \n",
       "3                           730            MA     Boston                    0   \n",
       "4                           740            MA   Lawrence                    9   \n",
       "...                         ...           ...        ...                  ...   \n",
       "400690                      760            MA       Lynn                  120   \n",
       "400691                      748            MA       Lynn                  120   \n",
       "400692                      760            MA       Lynn                  120   \n",
       "400693                      760            MA       Lynn                  120   \n",
       "400694                      760            MA       Lynn                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "400690             False         False            False   \n",
       "400691             False         False            False   \n",
       "400692             False         False            False   \n",
       "400693             False         False            False   \n",
       "400694             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "400690                       Gas  \n",
       "400691                       Gas  \n",
       "400692                       Gas  \n",
       "400693                       Gas  \n",
       "400694                       Gas  \n",
       "\n",
       "[360677 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/MA/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/MA-day/2018-aug-day-MA.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6582ec808432182d230d83f38f953be9df8c09ce</td>\n",
       "      <td>2018-08-01 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>765</td>\n",
       "      <td>645</td>\n",
       "      <td>MA</td>\n",
       "      <td>Tewksbury</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>34bcd16b28ca18aecb1ccc7a24eb5b1c78c4a74a</td>\n",
       "      <td>2018-08-14 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>744</td>\n",
       "      <td>750</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Hampden</td>\n",
       "      <td>18</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>34bcd16b28ca18aecb1ccc7a24eb5b1c78c4a74a</td>\n",
       "      <td>2018-08-27 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>MA</td>\n",
       "      <td>Hampden</td>\n",
       "      <td>18</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0ec218045f34ba636a652384c8c0083f0c461e88</td>\n",
       "      <td>2018-08-01 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>MA</td>\n",
       "      <td>Billerica</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1fe369fb7ef925bf587420fffd6d5c2fcdb653d6</td>\n",
       "      <td>2018-08-26 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Dunstable</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987959</th>\n",
       "      <td>2961eda333c6abfc8111d62235ad6dac2f3ce57a</td>\n",
       "      <td>2018-08-07 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>789</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Fitchburg</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987960</th>\n",
       "      <td>06b11f6f82efe99b19192c65670ab6653a93ea90</td>\n",
       "      <td>2018-08-08 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987961</th>\n",
       "      <td>f72bd16584f430a000cd0586fbe83212553cb2d6</td>\n",
       "      <td>2018-08-14 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>785</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lynn</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987962</th>\n",
       "      <td>c069e95a8b1f5d3e9be31e5d883360d56b7473b5</td>\n",
       "      <td>2018-08-28 09:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Swansea</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>987963</th>\n",
       "      <td>d03f842a6ef781d94451c0d1252fa321d90b054b</td>\n",
       "      <td>2018-08-04 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Brookline</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>886152 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6582ec808432182d230d83f38f953be9df8c09ce  2018-08-01 18:25:00 UTC   \n",
       "1       34bcd16b28ca18aecb1ccc7a24eb5b1c78c4a74a  2018-08-14 12:15:00 UTC   \n",
       "3       34bcd16b28ca18aecb1ccc7a24eb5b1c78c4a74a  2018-08-27 18:55:00 UTC   \n",
       "4       0ec218045f34ba636a652384c8c0083f0c461e88  2018-08-01 14:00:00 UTC   \n",
       "5       1fe369fb7ef925bf587420fffd6d5c2fcdb653d6  2018-08-26 18:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "987959  2961eda333c6abfc8111d62235ad6dac2f3ce57a  2018-08-07 19:20:00 UTC   \n",
       "987960  06b11f6f82efe99b19192c65670ab6653a93ea90  2018-08-08 13:50:00 UTC   \n",
       "987961  f72bd16584f430a000cd0586fbe83212553cb2d6  2018-08-14 17:45:00 UTC   \n",
       "987962  c069e95a8b1f5d3e9be31e5d883360d56b7473b5  2018-08-28 09:55:00 UTC   \n",
       "987963  d03f842a6ef781d94451c0d1252fa321d90b054b  2018-08-04 16:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               763                      765   \n",
       "1          cool          auto               744                      750   \n",
       "3          cool          hold               735                      725   \n",
       "4          cool          hold               730                      800   \n",
       "5          cool          hold               709                      705   \n",
       "...         ...           ...               ...                      ...   \n",
       "987959     cool          hold               789                      760   \n",
       "987960     cool          hold               759                      760   \n",
       "987961     cool          hold               785                      780   \n",
       "987962     cool          hold               758                      760   \n",
       "987963     cool          auto               739                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           645            MA  Tewksbury                   20   \n",
       "1                           685            MA    Hampden                   18   \n",
       "3                           725            MA    Hampden                   18   \n",
       "4                           800            MA  Billerica                   57   \n",
       "5                           705            MA  Dunstable                    5   \n",
       "...                         ...           ...        ...                  ...   \n",
       "987959                      760            MA  Fitchburg                   60   \n",
       "987960                      760            MA    andover                    0   \n",
       "987961                      760            MA       Lynn                  120   \n",
       "987962                      760            MA    Swansea                   30   \n",
       "987963                      760            MA  Brookline                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "5                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "987959             False         False            False   \n",
       "987960             False         False            False   \n",
       "987961             False         False            False   \n",
       "987962              True         False            False   \n",
       "987963              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "987959                       Gas  \n",
       "987960                       Gas  \n",
       "987961                       Gas  \n",
       "987962                       Gas  \n",
       "987963                       Gas  \n",
       "\n",
       "[886152 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/MA/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/MA-day/2019-aug-day-MA.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ba5e6315d62315cea9525ba33c92723eca1400f2</td>\n",
       "      <td>2019-08-29 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>747</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Springfield</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dc689991641ec6c0367705a72c31c5c8dc57a55b</td>\n",
       "      <td>2019-08-14 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5fdac0f56169ddd257884f4452f8958d42d91de1</td>\n",
       "      <td>2019-08-31 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>MA</td>\n",
       "      <td>Southborough</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a187f3658ba313f5e4a8015c00fe7cd6e061a8ca</td>\n",
       "      <td>2019-08-08 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>MA</td>\n",
       "      <td>Salem</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>95b6f6d10db6f24db56877600e5998598834ff08</td>\n",
       "      <td>2019-08-11 13:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>705</td>\n",
       "      <td>645</td>\n",
       "      <td>MA</td>\n",
       "      <td>Medford</td>\n",
       "      <td>115</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202671</th>\n",
       "      <td>32da4bccbf261ada336773d577f727effae2ea94</td>\n",
       "      <td>2019-08-25 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>West Bridgewater</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202672</th>\n",
       "      <td>f9e817cb19370d42f858d78290cba0257e4c1ca2</td>\n",
       "      <td>2019-08-17 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Milford</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202673</th>\n",
       "      <td>adc3c6a7b8ced2ec1dd39e1943bb28129eb0c148</td>\n",
       "      <td>2019-08-02 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Marshfield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202674</th>\n",
       "      <td>fa4d6c93022e9e972ebc80cbe2ca30f1eef8779c</td>\n",
       "      <td>2019-08-12 10:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Westborough</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202675</th>\n",
       "      <td>3a1e48068f45cd2e8d9f94c820c55edb739a0a99</td>\n",
       "      <td>2019-08-19 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Norton</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1072885 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        ba5e6315d62315cea9525ba33c92723eca1400f2  2019-08-29 14:50:00 UTC   \n",
       "2        dc689991641ec6c0367705a72c31c5c8dc57a55b  2019-08-14 17:30:00 UTC   \n",
       "3        5fdac0f56169ddd257884f4452f8958d42d91de1  2019-08-31 15:10:00 UTC   \n",
       "5        a187f3658ba313f5e4a8015c00fe7cd6e061a8ca  2019-08-08 12:10:00 UTC   \n",
       "7        95b6f6d10db6f24db56877600e5998598834ff08  2019-08-11 13:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1202671  32da4bccbf261ada336773d577f727effae2ea94  2019-08-25 16:20:00 UTC   \n",
       "1202672  f9e817cb19370d42f858d78290cba0257e4c1ca2  2019-08-17 19:30:00 UTC   \n",
       "1202673  adc3c6a7b8ced2ec1dd39e1943bb28129eb0c148  2019-08-02 17:30:00 UTC   \n",
       "1202674  fa4d6c93022e9e972ebc80cbe2ca30f1eef8779c  2019-08-12 10:55:00 UTC   \n",
       "1202675  3a1e48068f45cd2e8d9f94c820c55edb739a0a99  2019-08-19 19:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               754                      750   \n",
       "2           cool          hold               742                      745   \n",
       "3           cool          auto               734                      734   \n",
       "5           cool          hold               747                      745   \n",
       "7           auto          hold               706                      705   \n",
       "...          ...           ...               ...                      ...   \n",
       "1202671     cool          hold               725                      760   \n",
       "1202672     cool          hold               763                      760   \n",
       "1202673     cool          hold               754                      760   \n",
       "1202674     cool          hold               730                      760   \n",
       "1202675     cool          hold               759                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            747            MA  West Springfield   \n",
       "2                            745            MA            Natick   \n",
       "3                            734            MA      Southborough   \n",
       "5                            745            MA             Salem   \n",
       "7                            645            MA           Medford   \n",
       "...                          ...           ...               ...   \n",
       "1202671                      760            MA  West Bridgewater   \n",
       "1202672                      760            MA           Milford   \n",
       "1202673                      760            MA        Marshfield   \n",
       "1202674                      760            MA       Westborough   \n",
       "1202675                      760            MA            Norton   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "2                        117             False         False            False   \n",
       "3                         20             False         False            False   \n",
       "5                          0              True         False             True   \n",
       "7                        115             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1202671                   40             False         False            False   \n",
       "1202672                   39              True         False            False   \n",
       "1202673                    0             False         False            False   \n",
       "1202674                   20             False         False            False   \n",
       "1202675                   10              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "5                        Electric  \n",
       "7                             Gas  \n",
       "...                           ...  \n",
       "1202671                       Gas  \n",
       "1202672                       Gas  \n",
       "1202673                       Gas  \n",
       "1202674                       Gas  \n",
       "1202675                       Gas  \n",
       "\n",
       "[1072885 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/MA/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/MA-day/2020-aug-day-MA.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62e97b3049c26cccf2bd1219ad1392169cc58a05</td>\n",
       "      <td>2020-08-18 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5f8eca8283628c60a55c2a4e869e1a37581de65b</td>\n",
       "      <td>2020-08-29 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>717</td>\n",
       "      <td>717</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e56212e2ddecb27c59eaa6464f08a05be98dc947</td>\n",
       "      <td>2020-08-18 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>MA</td>\n",
       "      <td>Belmont</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e023c1e547bf7db96e001007be7b1dd244f418ff</td>\n",
       "      <td>2020-08-26 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>MA</td>\n",
       "      <td>Waltham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cd4c87b39e30363018921df9f70f38310673248f</td>\n",
       "      <td>2020-08-21 11:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>MA</td>\n",
       "      <td>Watertown</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262179</th>\n",
       "      <td>8e174c8b4cba7c2b65671257aea9b99ba650e848</td>\n",
       "      <td>2020-08-13 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Winchester</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262180</th>\n",
       "      <td>e5a996e4772f2715b33bbbf3f623cae7c06ab044</td>\n",
       "      <td>2020-08-15 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Dedham</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262181</th>\n",
       "      <td>b0a85c638a961bf0f52136290a5b9e864c864fef</td>\n",
       "      <td>2020-08-11 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Walpole</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262182</th>\n",
       "      <td>8956817d075ed2d702e401618cc93bdbe3d2eb8a</td>\n",
       "      <td>2020-08-12 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Bolton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1262183</th>\n",
       "      <td>d02953442bd2f59df34cf943fce14557e48c3303</td>\n",
       "      <td>2020-08-30 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Newton</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1105756 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        62e97b3049c26cccf2bd1219ad1392169cc58a05  2020-08-18 18:55:00 UTC   \n",
       "1        5f8eca8283628c60a55c2a4e869e1a37581de65b  2020-08-29 14:00:00 UTC   \n",
       "2        e56212e2ddecb27c59eaa6464f08a05be98dc947  2020-08-18 12:30:00 UTC   \n",
       "3        e023c1e547bf7db96e001007be7b1dd244f418ff  2020-08-26 18:55:00 UTC   \n",
       "4        cd4c87b39e30363018921df9f70f38310673248f  2020-08-21 11:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1262179  8e174c8b4cba7c2b65671257aea9b99ba650e848  2020-08-13 12:15:00 UTC   \n",
       "1262180  e5a996e4772f2715b33bbbf3f623cae7c06ab044  2020-08-15 17:55:00 UTC   \n",
       "1262181  b0a85c638a961bf0f52136290a5b9e864c864fef  2020-08-11 12:10:00 UTC   \n",
       "1262182  8956817d075ed2d702e401618cc93bdbe3d2eb8a  2020-08-12 15:15:00 UTC   \n",
       "1262183  d02953442bd2f59df34cf943fce14557e48c3303  2020-08-30 19:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               740                      725   \n",
       "1           cool          hold               683                      717   \n",
       "2           cool          hold               676                      675   \n",
       "3           cool          hold               761                      749   \n",
       "4           cool          hold               708                      719   \n",
       "...          ...           ...               ...                      ...   \n",
       "1262179     cool          hold               757                      760   \n",
       "1262180     cool          hold               717                      760   \n",
       "1262181     cool          hold               756                      760   \n",
       "1262182     cool          hold               761                      760   \n",
       "1262183     cool          hold               770                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                            725            MA   Cambridge   \n",
       "1                            717            MA     Andover   \n",
       "2                            675            MA     Belmont   \n",
       "3                            749            MA     Waltham   \n",
       "4                            719            MA   Watertown   \n",
       "...                          ...           ...         ...   \n",
       "1262179                      760            MA  Winchester   \n",
       "1262180                      760            MA      Dedham   \n",
       "1262181                      760            MA     Walpole   \n",
       "1262182                      760            MA      Bolton   \n",
       "1262183                      760            MA      Newton   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        120              True         False            False   \n",
       "1                          0             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1262179                   10              True         False            False   \n",
       "1262180                    5             False         False            False   \n",
       "1262181                   10             False         False            False   \n",
       "1262182                    5             False         False            False   \n",
       "1262183                    1             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1262179                       Gas  \n",
       "1262180                       Gas  \n",
       "1262181                       Gas  \n",
       "1262182                       Gas  \n",
       "1262183                       Gas  \n",
       "\n",
       "[1105756 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/MA/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>737.961538</td>\n",
       "      <td>750.269231</td>\n",
       "      <td>749.500000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02382555cf0333ff1ad92861205814fd74616f88</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Hadley</td>\n",
       "      <td>784.111111</td>\n",
       "      <td>774.828283</td>\n",
       "      <td>679.818182</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boston</td>\n",
       "      <td>732.733333</td>\n",
       "      <td>733.434457</td>\n",
       "      <td>645.138577</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0373b135029a70f677ca2ad36469b0088d3c56bf</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>742.904110</td>\n",
       "      <td>750.212329</td>\n",
       "      <td>750.212329</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1732</th>\n",
       "      <td>ff19754ee5d7c1c54c2d4d54707dd93ef9d562af</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Reading</td>\n",
       "      <td>721.576087</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1733</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>779.262136</td>\n",
       "      <td>798.533981</td>\n",
       "      <td>673.640777</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1734</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>760.155235</td>\n",
       "      <td>747.610108</td>\n",
       "      <td>747.393502</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1735</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Salem</td>\n",
       "      <td>725.111111</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1736</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>717.289855</td>\n",
       "      <td>710.420290</td>\n",
       "      <td>710.072464</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5703 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   aug  2017     cool   \n",
       "1     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   aug  2017     cool   \n",
       "2     02382555cf0333ff1ad92861205814fd74616f88   aug  2017     cool   \n",
       "3     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   aug  2017     auto   \n",
       "4     0373b135029a70f677ca2ad36469b0088d3c56bf   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1732  ff19754ee5d7c1c54c2d4d54707dd93ef9d562af   aug  2020     cool   \n",
       "1733  fff0787bec7315864a164645420fade19da92725   aug  2020     cool   \n",
       "1734  fff0787bec7315864a164645420fade19da92725   aug  2020     cool   \n",
       "1735  fffa05184379d65eec4deeb00b14374669d0ff1e   aug  2020     cool   \n",
       "1736  fffa05184379d65eec4deeb00b14374669d0ff1e   aug  2020     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto      Medfield            770.000000   \n",
       "1             hold      Medfield            737.961538   \n",
       "2             hold  South Hadley            784.111111   \n",
       "3             hold        Boston            732.733333   \n",
       "4             hold     Hopkinton            742.904110   \n",
       "...            ...           ...                   ...   \n",
       "1732          hold       Reading            721.576087   \n",
       "1733          auto        Agawam            779.262136   \n",
       "1734          hold        Agawam            760.155235   \n",
       "1735          auto         Salem            725.111111   \n",
       "1736          hold         Salem            717.289855   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      780.000000                   680.000000   \n",
       "1                      750.269231                   749.500000   \n",
       "2                      774.828283                   679.818182   \n",
       "3                      733.434457                   645.138577   \n",
       "4                      750.212329                   750.212329   \n",
       "...                           ...                          ...   \n",
       "1732                   690.000000                   690.000000   \n",
       "1733                   798.533981                   673.640777   \n",
       "1734                   747.610108                   747.393502   \n",
       "1735                   720.000000                   720.000000   \n",
       "1736                   710.420290                   710.072464   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    37.0             False         False            False  \n",
       "1                    37.0             False         False            False  \n",
       "2                    55.0             False         False            False  \n",
       "3                   100.0             False         False            False  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1732                  0.0             False         False            False  \n",
       "1733                 95.0             False         False            False  \n",
       "1734                 95.0             False         False            False  \n",
       "1735                  6.0             False         False            False  \n",
       "1736                  6.0             False         False            False  \n",
       "\n",
       "[5703 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/aug/\" + file)\n",
    "    MA_aug = pd.concat([MA_aug, df])\n",
    "    \n",
    "MA_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_aug.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/MA-day/2017-dec-day-MA.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>50a31ab86186627f55d252389c7cb7ec36c669e8</td>\n",
       "      <td>2017-12-17 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>768</td>\n",
       "      <td>651</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wrentham</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9c8b53bdd50859c24aab6bbbca68bc7f6db011b3</td>\n",
       "      <td>2017-12-20 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>MA</td>\n",
       "      <td>Scituate</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b3b2f42a448e364eaa79a51ff29583a162421152</td>\n",
       "      <td>2017-12-17 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Wrentham</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4f61cf385a76917e48647c46f8bcf8d521d31204</td>\n",
       "      <td>2017-12-15 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>723</td>\n",
       "      <td>723</td>\n",
       "      <td>MA</td>\n",
       "      <td>Methuen</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>268ed930f8c7a70709b09cb843ead0fc45a6dbce</td>\n",
       "      <td>2017-12-03 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stoneham</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772428</th>\n",
       "      <td>b2bc7cf878738760725ddb96d8e183769c2d1e5f</td>\n",
       "      <td>2017-12-25 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Uxbridge</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772429</th>\n",
       "      <td>b2bc7cf878738760725ddb96d8e183769c2d1e5f</td>\n",
       "      <td>2017-12-09 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Uxbridge</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772430</th>\n",
       "      <td>b2bc7cf878738760725ddb96d8e183769c2d1e5f</td>\n",
       "      <td>2017-12-27 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Uxbridge</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772431</th>\n",
       "      <td>b2bc7cf878738760725ddb96d8e183769c2d1e5f</td>\n",
       "      <td>2017-12-25 11:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Uxbridge</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772432</th>\n",
       "      <td>b2bc7cf878738760725ddb96d8e183769c2d1e5f</td>\n",
       "      <td>2017-12-28 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>MA</td>\n",
       "      <td>Uxbridge</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>721255 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       50a31ab86186627f55d252389c7cb7ec36c669e8  2017-12-17 16:05:00 UTC   \n",
       "1       9c8b53bdd50859c24aab6bbbca68bc7f6db011b3  2017-12-20 16:15:00 UTC   \n",
       "2       b3b2f42a448e364eaa79a51ff29583a162421152  2017-12-17 16:35:00 UTC   \n",
       "3       4f61cf385a76917e48647c46f8bcf8d521d31204  2017-12-15 14:00:00 UTC   \n",
       "4       268ed930f8c7a70709b09cb843ead0fc45a6dbce  2017-12-03 17:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "772428  b2bc7cf878738760725ddb96d8e183769c2d1e5f  2017-12-25 12:35:00 UTC   \n",
       "772429  b2bc7cf878738760725ddb96d8e183769c2d1e5f  2017-12-09 14:25:00 UTC   \n",
       "772430  b2bc7cf878738760725ddb96d8e183769c2d1e5f  2017-12-27 13:45:00 UTC   \n",
       "772431  b2bc7cf878738760725ddb96d8e183769c2d1e5f  2017-12-25 11:50:00 UTC   \n",
       "772432  b2bc7cf878738760725ddb96d8e183769c2d1e5f  2017-12-28 16:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               661                      768   \n",
       "1          heat          hold               668                      640   \n",
       "2          heat          hold               663                      685   \n",
       "3          heat          hold               719                      723   \n",
       "4          auto          hold               683                      735   \n",
       "...         ...           ...               ...                      ...   \n",
       "772428     auto          hold               701                      765   \n",
       "772429     auto          hold               691                      765   \n",
       "772430     auto          hold               696                      765   \n",
       "772431     auto          hold               689                      765   \n",
       "772432     auto          hold               690                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           651            MA  Wrentham                   55   \n",
       "1                           640            MA  Scituate                   35   \n",
       "2                           685            MA  Wrentham                    0   \n",
       "3                           723            MA   Methuen                   45   \n",
       "4                           685            MA  Stoneham                   66   \n",
       "...                         ...           ...       ...                  ...   \n",
       "772428                      695            MA  Uxbridge                   57   \n",
       "772429                      695            MA  Uxbridge                   57   \n",
       "772430                      695            MA  Uxbridge                   57   \n",
       "772431                      695            MA  Uxbridge                   57   \n",
       "772432                      695            MA  Uxbridge                   57   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False             True   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "772428             False         False             True   \n",
       "772429             False         False             True   \n",
       "772430             False         False             True   \n",
       "772431             False         False             True   \n",
       "772432             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "772428                  Electric  \n",
       "772429                  Electric  \n",
       "772430                  Electric  \n",
       "772431                  Electric  \n",
       "772432                  Electric  \n",
       "\n",
       "[721255 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/MA/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/MA-day/2018-dec-day-MA.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>43606b50a9fbf5bb38e62ee60e36325f9ca2b7c6</td>\n",
       "      <td>2018-12-09 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>702</td>\n",
       "      <td>652</td>\n",
       "      <td>MA</td>\n",
       "      <td>Medford</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6582ec808432182d230d83f38f953be9df8c09ce</td>\n",
       "      <td>2018-12-02 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>735</td>\n",
       "      <td>675</td>\n",
       "      <td>MA</td>\n",
       "      <td>Tewksbury</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>515dbb4dbde20e0fe5e8e9892c69e23d3c54e33c</td>\n",
       "      <td>2018-12-11 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>778</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>Millis</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>675e97e14602b3adfee227ef565c0205a1f0d40c</td>\n",
       "      <td>2018-12-04 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Yarmouth port</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7da25a79561e50ffe514f57f9e83e77671e012cb</td>\n",
       "      <td>2018-12-10 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Stoughton</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296350</th>\n",
       "      <td>b2691cdb2fa97bf81777cd4d99232d3f75c300b9</td>\n",
       "      <td>2018-12-23 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Attleboro</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296351</th>\n",
       "      <td>314bc12abae59de0d70dce0595bc58d16af08366</td>\n",
       "      <td>2018-12-16 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296352</th>\n",
       "      <td>314bc12abae59de0d70dce0595bc58d16af08366</td>\n",
       "      <td>2018-12-06 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Cambridge</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296353</th>\n",
       "      <td>f59ec0451a2edb3c82251e60c6e2124849fa1286</td>\n",
       "      <td>2018-12-19 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>678</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>MA</td>\n",
       "      <td>Abington</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1296354</th>\n",
       "      <td>a2fe70a7fe1306acd2e2953fa9c79e29077ac8bf</td>\n",
       "      <td>2018-12-01 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Belmont</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1138572 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        43606b50a9fbf5bb38e62ee60e36325f9ca2b7c6  2018-12-09 15:00:00 UTC   \n",
       "1        6582ec808432182d230d83f38f953be9df8c09ce  2018-12-02 19:10:00 UTC   \n",
       "2        515dbb4dbde20e0fe5e8e9892c69e23d3c54e33c  2018-12-11 13:00:00 UTC   \n",
       "3        675e97e14602b3adfee227ef565c0205a1f0d40c  2018-12-04 14:45:00 UTC   \n",
       "4        7da25a79561e50ffe514f57f9e83e77671e012cb  2018-12-10 12:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1296350  b2691cdb2fa97bf81777cd4d99232d3f75c300b9  2018-12-23 16:00:00 UTC   \n",
       "1296351  314bc12abae59de0d70dce0595bc58d16af08366  2018-12-16 18:05:00 UTC   \n",
       "1296352  314bc12abae59de0d70dce0595bc58d16af08366  2018-12-06 16:10:00 UTC   \n",
       "1296353  f59ec0451a2edb3c82251e60c6e2124849fa1286  2018-12-19 11:35:00 UTC   \n",
       "1296354  a2fe70a7fe1306acd2e2953fa9c79e29077ac8bf  2018-12-01 12:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               677                      702   \n",
       "1           auto          hold               678                      735   \n",
       "2           auto          auto               718                      778   \n",
       "3           heat          hold               688                      698   \n",
       "4           auto          hold               704                      755   \n",
       "...          ...           ...               ...                      ...   \n",
       "1296350     auto          hold               707                      760   \n",
       "1296351     heat          auto               706                      760   \n",
       "1296352     heat          auto               697                      760   \n",
       "1296353     heat          auto               678                      760   \n",
       "1296354     auto          hold               690                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            652            MA        Medford   \n",
       "1                            675            MA      Tewksbury   \n",
       "2                            720            MA         Millis   \n",
       "3                            698            MA  Yarmouth port   \n",
       "4                            705            MA      Stoughton   \n",
       "...                          ...           ...            ...   \n",
       "1296350                      710            MA      Attleboro   \n",
       "1296351                      710            MA      Cambridge   \n",
       "1296352                      700            MA      Cambridge   \n",
       "1296353                      680            MA       Abington   \n",
       "1296354                      700            MA        Belmont   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         20             False         False            False   \n",
       "2                         15             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "4                         20             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1296350                    0             False         False            False   \n",
       "1296351                  120             False         False            False   \n",
       "1296352                  120             False         False            False   \n",
       "1296353                   17             False         False            False   \n",
       "1296354                   80             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1296350                       Gas  \n",
       "1296351                       Gas  \n",
       "1296352                       Gas  \n",
       "1296353                       Gas  \n",
       "1296354                       Gas  \n",
       "\n",
       "[1138572 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/MA/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/MA-day/2019-dec-day-MA.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0e76511b11c6f38465ad968c99410de1dae1682b</td>\n",
       "      <td>2019-12-20 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>613</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>MA</td>\n",
       "      <td>Andover</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dbb206a77b98b58451af3974bfd9608dd6824225</td>\n",
       "      <td>2019-12-15 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>679</td>\n",
       "      <td>679</td>\n",
       "      <td>MA</td>\n",
       "      <td>Gloucester</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>95b6f6d10db6f24db56877600e5998598834ff08</td>\n",
       "      <td>2019-12-20 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>MA</td>\n",
       "      <td>Medford</td>\n",
       "      <td>115</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>92ae0a5755a1acb9daac0a77df36e2d331984a1e</td>\n",
       "      <td>2019-12-19 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>MA</td>\n",
       "      <td>Acton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fa4c2205e4439e5d6300995412bfc5257160f0f5</td>\n",
       "      <td>2019-12-27 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>606</td>\n",
       "      <td>790</td>\n",
       "      <td>620</td>\n",
       "      <td>MA</td>\n",
       "      <td>Pepperell</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381633</th>\n",
       "      <td>7899fd8e4bf47513611cbb37e8fbd7afbd54c638</td>\n",
       "      <td>2019-12-28 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Quincy</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381634</th>\n",
       "      <td>1be0ae8ca4021a0cc7811cf692e21cf886f9d0a1</td>\n",
       "      <td>2019-12-10 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Lowell</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381635</th>\n",
       "      <td>66ef8c0b1d7397c741a5152bdf2f555e3cb9350e</td>\n",
       "      <td>2019-12-28 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381636</th>\n",
       "      <td>b35daa351e18ba1a8fd61de252be873eba95eb92</td>\n",
       "      <td>2019-12-24 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1381637</th>\n",
       "      <td>7b3b6469c609d9ecd69523f24599807605064c89</td>\n",
       "      <td>2019-12-17 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>696</td>\n",
       "      <td>MA</td>\n",
       "      <td>Middleborough</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1264342 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        0e76511b11c6f38465ad968c99410de1dae1682b  2019-12-20 15:55:00 UTC   \n",
       "1        dbb206a77b98b58451af3974bfd9608dd6824225  2019-12-15 18:05:00 UTC   \n",
       "2        95b6f6d10db6f24db56877600e5998598834ff08  2019-12-20 19:30:00 UTC   \n",
       "3        92ae0a5755a1acb9daac0a77df36e2d331984a1e  2019-12-19 13:45:00 UTC   \n",
       "4        fa4c2205e4439e5d6300995412bfc5257160f0f5  2019-12-27 14:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1381633  7899fd8e4bf47513611cbb37e8fbd7afbd54c638  2019-12-28 15:55:00 UTC   \n",
       "1381634  1be0ae8ca4021a0cc7811cf692e21cf886f9d0a1  2019-12-10 12:35:00 UTC   \n",
       "1381635  66ef8c0b1d7397c741a5152bdf2f555e3cb9350e  2019-12-28 17:30:00 UTC   \n",
       "1381636  b35daa351e18ba1a8fd61de252be873eba95eb92  2019-12-24 16:10:00 UTC   \n",
       "1381637  7b3b6469c609d9ecd69523f24599807605064c89  2019-12-17 11:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               613                      728   \n",
       "1           heat          hold               704                      679   \n",
       "2           heat          hold               695                      705   \n",
       "3           heat          hold               715                      707   \n",
       "4           heat          auto               606                      790   \n",
       "...          ...           ...               ...                      ...   \n",
       "1381633     auto          hold               699                      760   \n",
       "1381634     heat          hold               755                      760   \n",
       "1381635     auto          hold               705                      760   \n",
       "1381636     auto          hold               699                      760   \n",
       "1381637     auto          hold               694                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            728            MA        Andover   \n",
       "1                            679            MA     Gloucester   \n",
       "2                            705            MA        Medford   \n",
       "3                            707            MA          Acton   \n",
       "4                            620            MA      Pepperell   \n",
       "...                          ...           ...            ...   \n",
       "1381633                      710            MA         Quincy   \n",
       "1381634                      760            MA         Lowell   \n",
       "1381635                      690            MA         Boston   \n",
       "1381636                      700            MA         Boston   \n",
       "1381637                      696            MA  Middleborough   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "1                         30             False         False            False   \n",
       "2                        115             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1381633                  120             False         False            False   \n",
       "1381634                    5             False         False            False   \n",
       "1381635                    0             False         False             True   \n",
       "1381636                   95             False         False            False   \n",
       "1381637                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1381633                       Gas  \n",
       "1381634                       Gas  \n",
       "1381635                  Electric  \n",
       "1381636                       Gas  \n",
       "1381637                       Gas  \n",
       "\n",
       "[1264342 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/MA/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/MA-day/2020-dec-day-MA.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b421aa61a315121834909b5b59a0f28239f2f535</td>\n",
       "      <td>2020-12-13 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>631</td>\n",
       "      <td>630</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>86fbad6fa1796cc2e462bf50f248ed5f5ce20ef4</td>\n",
       "      <td>2020-12-02 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>MA</td>\n",
       "      <td>Saugus</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>90c1744d531225f3e14073c92c378d96206fcba0</td>\n",
       "      <td>2020-12-17 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>MA</td>\n",
       "      <td>Easthampton</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>620722c5a59ecd00708de17ec396d4066b987fed</td>\n",
       "      <td>2020-12-05 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>MA</td>\n",
       "      <td>Somerville</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1b7c81d681e222d8b3fb91002607ff8a129f3b3c</td>\n",
       "      <td>2020-12-02 12:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>MA</td>\n",
       "      <td>Natick</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272477</th>\n",
       "      <td>36e696300752b920e344d794735e188bf56001d7</td>\n",
       "      <td>2020-12-07 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Sudbury</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272478</th>\n",
       "      <td>492784f1c9ee5262dfd9147baf61632e236f02db</td>\n",
       "      <td>2020-12-02 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>MA</td>\n",
       "      <td>Malden</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272479</th>\n",
       "      <td>0541b0f1875c110880572e68cb7f96b08a0622b1</td>\n",
       "      <td>2020-12-20 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>MA</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272480</th>\n",
       "      <td>a0e78aef6400dbdae5e06462bb924a4de07faf7a</td>\n",
       "      <td>2020-12-10 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>MA</td>\n",
       "      <td>Littleton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1272481</th>\n",
       "      <td>2a967eb9e73bdc7131c5ff22245767f2c9ec5e55</td>\n",
       "      <td>2020-12-20 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>MA</td>\n",
       "      <td>Boston</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1191346 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        b421aa61a315121834909b5b59a0f28239f2f535  2020-12-13 19:30:00 UTC   \n",
       "1        86fbad6fa1796cc2e462bf50f248ed5f5ce20ef4  2020-12-02 13:50:00 UTC   \n",
       "2        90c1744d531225f3e14073c92c378d96206fcba0  2020-12-17 19:50:00 UTC   \n",
       "3        620722c5a59ecd00708de17ec396d4066b987fed  2020-12-05 12:05:00 UTC   \n",
       "4        1b7c81d681e222d8b3fb91002607ff8a129f3b3c  2020-12-02 12:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1272477  36e696300752b920e344d794735e188bf56001d7  2020-12-07 15:15:00 UTC   \n",
       "1272478  492784f1c9ee5262dfd9147baf61632e236f02db  2020-12-02 14:10:00 UTC   \n",
       "1272479  0541b0f1875c110880572e68cb7f96b08a0622b1  2020-12-20 13:25:00 UTC   \n",
       "1272480  a0e78aef6400dbdae5e06462bb924a4de07faf7a  2020-12-10 19:05:00 UTC   \n",
       "1272481  2a967eb9e73bdc7131c5ff22245767f2c9ec5e55  2020-12-20 13:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               671                      631   \n",
       "1           auto          hold               714                      765   \n",
       "2           heat          hold               667                      655   \n",
       "3           heat          hold               691                      698   \n",
       "4           auto          hold               696                      735   \n",
       "...          ...           ...               ...                      ...   \n",
       "1272477     auto          hold               703                      760   \n",
       "1272478     cool          hold               664                      760   \n",
       "1272479     auto          auto               700                      760   \n",
       "1272480     heat          auto               727                      760   \n",
       "1272481     auto          hold               708                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            630            MA       Natick   \n",
       "1                            715            MA       Saugus   \n",
       "2                            655            MA  Easthampton   \n",
       "3                            698            MA   Somerville   \n",
       "4                            685            MA       Natick   \n",
       "...                          ...           ...          ...   \n",
       "1272477                      700            MA      Sudbury   \n",
       "1272478                      760            MA       Malden   \n",
       "1272479                      700            MA  Southampton   \n",
       "1272480                      720            MA    Littleton   \n",
       "1272481                      710            MA       Boston   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                          0              True         False            False   \n",
       "2                         45             False         False            False   \n",
       "3                         90             False         False            False   \n",
       "4                         27             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1272477                   10             False         False            False   \n",
       "1272478                    5              True         False            False   \n",
       "1272479                   17             False         False            False   \n",
       "1272480                    0             False         False            False   \n",
       "1272481                   30             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1272477                       Gas  \n",
       "1272478                       Gas  \n",
       "1272479                       Gas  \n",
       "1272480                       Gas  \n",
       "1272481                       Gas  \n",
       "\n",
       "[1191346 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/MA/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/MA/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>645.422222</td>\n",
       "      <td>650.866667</td>\n",
       "      <td>636.600000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>680.053299</td>\n",
       "      <td>652.616751</td>\n",
       "      <td>648.109137</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01dddc9a4241063b0f3fe22c1cbbd66cd3c8ca16</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Amesbury</td>\n",
       "      <td>678.723440</td>\n",
       "      <td>680.091062</td>\n",
       "      <td>680.091062</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01dddc9a4241063b0f3fe22c1cbbd66cd3c8ca16</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Amesbury</td>\n",
       "      <td>679.095949</td>\n",
       "      <td>677.292111</td>\n",
       "      <td>677.072495</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>021bc49d2737cfc5291795252754c8c84d99fb90</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Plainville</td>\n",
       "      <td>702.805785</td>\n",
       "      <td>705.632231</td>\n",
       "      <td>705.355372</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2113</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Westford</td>\n",
       "      <td>657.407051</td>\n",
       "      <td>703.333333</td>\n",
       "      <td>656.156517</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2114</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>661.088235</td>\n",
       "      <td>820.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2115</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>678.500000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2116</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Salem</td>\n",
       "      <td>712.680000</td>\n",
       "      <td>715.146667</td>\n",
       "      <td>714.826667</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2117</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>702.236697</td>\n",
       "      <td>707.814679</td>\n",
       "      <td>707.546789</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8350 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   dec  2017     heat   \n",
       "1     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   dec  2017     heat   \n",
       "2     01dddc9a4241063b0f3fe22c1cbbd66cd3c8ca16   dec  2017     heat   \n",
       "3     01dddc9a4241063b0f3fe22c1cbbd66cd3c8ca16   dec  2017     heat   \n",
       "4     021bc49d2737cfc5291795252754c8c84d99fb90   dec  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2113  feb0e1fc76a69b0d234d4874cf27bae767b20c81   dec  2020     heat   \n",
       "2114  fff0787bec7315864a164645420fade19da92725   dec  2020     heat   \n",
       "2115  fff0787bec7315864a164645420fade19da92725   dec  2020     heat   \n",
       "2116  fffa05184379d65eec4deeb00b14374669d0ff1e   dec  2020     heat   \n",
       "2117  fffa05184379d65eec4deeb00b14374669d0ff1e   dec  2020     heat   \n",
       "\n",
       "     CalendarEvent        City  Temperature_ctrl_ave  \\\n",
       "0             auto    Medfield            645.422222   \n",
       "1             hold    Medfield            680.053299   \n",
       "2             auto    Amesbury            678.723440   \n",
       "3             hold    Amesbury            679.095949   \n",
       "4             auto  Plainville            702.805785   \n",
       "...            ...         ...                   ...   \n",
       "2113          auto    Westford            657.407051   \n",
       "2114          auto      Agawam            661.088235   \n",
       "2115          hold      Agawam            678.500000   \n",
       "2116          auto       Salem            712.680000   \n",
       "2117          hold       Salem            702.236697   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      650.866667                   636.600000   \n",
       "1                      652.616751                   648.109137   \n",
       "2                      680.091062                   680.091062   \n",
       "3                      677.292111                   677.072495   \n",
       "4                      705.632231                   705.355372   \n",
       "...                           ...                          ...   \n",
       "2113                   703.333333                   656.156517   \n",
       "2114                   820.000000                   640.000000   \n",
       "2115                   690.000000                   690.000000   \n",
       "2116                   715.146667                   714.826667   \n",
       "2117                   707.814679                   707.546789   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    37.0             False         False            False  \n",
       "1                    37.0             False         False            False  \n",
       "2                     7.0             False         False            False  \n",
       "3                     7.0             False         False            False  \n",
       "4                    35.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2113                 30.0              True         False            False  \n",
       "2114                 95.0             False         False            False  \n",
       "2115                 95.0             False         False            False  \n",
       "2116                  6.0             False         False            False  \n",
       "2117                  6.0             False         False            False  \n",
       "\n",
       "[8350 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/MA/dec/\" + file)\n",
    "    MA_dec = pd.concat([MA_dec, df])\n",
    "    \n",
    "MA_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_dec.to_csv(\"Scraper_Output/State_Month_Day/MA/MA_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/MA/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medfield</td>\n",
       "      <td>737.961538</td>\n",
       "      <td>750.269231</td>\n",
       "      <td>749.500000</td>\n",
       "      <td>37.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02382555cf0333ff1ad92861205814fd74616f88</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>South Hadley</td>\n",
       "      <td>784.111111</td>\n",
       "      <td>774.828283</td>\n",
       "      <td>679.818182</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02fe14bbcc60780d648cd5eae6868f1ff7e79b3c</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boston</td>\n",
       "      <td>732.733333</td>\n",
       "      <td>733.434457</td>\n",
       "      <td>645.138577</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0373b135029a70f677ca2ad36469b0088d3c56bf</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hopkinton</td>\n",
       "      <td>742.904110</td>\n",
       "      <td>750.212329</td>\n",
       "      <td>750.212329</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7780</th>\n",
       "      <td>fe2bdeb6fe92670a4befad74d25a7061473e51f8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Norwood</td>\n",
       "      <td>736.869565</td>\n",
       "      <td>733.000000</td>\n",
       "      <td>683.000000</td>\n",
       "      <td>67.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7781</th>\n",
       "      <td>fe3de6f77ca9ec90aab29b2a5918552e6ff9229e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westborough</td>\n",
       "      <td>737.575436</td>\n",
       "      <td>734.186145</td>\n",
       "      <td>734.186145</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7782</th>\n",
       "      <td>feb0e1fc76a69b0d234d4874cf27bae767b20c81</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westford</td>\n",
       "      <td>698.208333</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7783</th>\n",
       "      <td>fff0787bec7315864a164645420fade19da92725</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Agawam</td>\n",
       "      <td>762.393939</td>\n",
       "      <td>739.090909</td>\n",
       "      <td>739.090909</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7784</th>\n",
       "      <td>fffa05184379d65eec4deeb00b14374669d0ff1e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>713.350746</td>\n",
       "      <td>712.985075</td>\n",
       "      <td>712.985075</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44651 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   aug  2017     cool   \n",
       "1     00a0f0ba8d5a1f94ea2806afb6c1baa70177c6c1   aug  2017     cool   \n",
       "2     02382555cf0333ff1ad92861205814fd74616f88   aug  2017     cool   \n",
       "3     02fe14bbcc60780d648cd5eae6868f1ff7e79b3c   aug  2017     auto   \n",
       "4     0373b135029a70f677ca2ad36469b0088d3c56bf   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "7780  fe2bdeb6fe92670a4befad74d25a7061473e51f8   jun  2021     auto   \n",
       "7781  fe3de6f77ca9ec90aab29b2a5918552e6ff9229e   jun  2021     cool   \n",
       "7782  feb0e1fc76a69b0d234d4874cf27bae767b20c81   jun  2021     cool   \n",
       "7783  fff0787bec7315864a164645420fade19da92725   jun  2021     cool   \n",
       "7784  fffa05184379d65eec4deeb00b14374669d0ff1e   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto      Medfield            770.000000   \n",
       "1             hold      Medfield            737.961538   \n",
       "2             hold  South Hadley            784.111111   \n",
       "3             hold        Boston            732.733333   \n",
       "4             hold     Hopkinton            742.904110   \n",
       "...            ...           ...                   ...   \n",
       "7780          hold       Norwood            736.869565   \n",
       "7781          hold   Westborough            737.575436   \n",
       "7782          hold      Westford            698.208333   \n",
       "7783          hold        Agawam            762.393939   \n",
       "7784          hold         Salem            713.350746   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      780.000000                   680.000000   \n",
       "1                      750.269231                   749.500000   \n",
       "2                      774.828283                   679.818182   \n",
       "3                      733.434457                   645.138577   \n",
       "4                      750.212329                   750.212329   \n",
       "...                           ...                          ...   \n",
       "7780                   733.000000                   683.000000   \n",
       "7781                   734.186145                   734.186145   \n",
       "7782                   695.000000                   695.000000   \n",
       "7783                   739.090909                   739.090909   \n",
       "7784                   712.985075                   712.985075   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    37.0             False         False            False  \n",
       "1                    37.0             False         False            False  \n",
       "2                    55.0             False         False            False  \n",
       "3                   100.0             False         False            False  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "7780                 67.0             False         False            False  \n",
       "7781                 20.0             False         False            False  \n",
       "7782                 30.0              True         False            False  \n",
       "7783                 95.0             False         False            False  \n",
       "7784                  6.0             False         False            False  \n",
       "\n",
       "[44651 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "MA_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/MA/\" + file)\n",
    "    MA_all = pd.concat([MA_all, df])\n",
    "    \n",
    "MA_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "MA_all.to_csv(\"Scraper_Output/State_Month_Day/MA_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['MA']\n",
      "Unique jan_2018: ['MA']\n",
      "Unique jan_2019: ['MA']\n",
      "Unique jan_2020: ['MA']\n",
      "Unique jan_2021: ['MA']\n",
      "Unique feb_2017: ['MA']\n",
      "Unique feb_2018: ['MA']\n",
      "Unique feb_2019: ['MA']\n",
      "Unique feb_2020: ['MA']\n",
      "Unique feb_2021: ['MA']\n",
      "Unique jun_2017: ['MA']\n",
      "Unique jun_2018: ['MA']\n",
      "Unique jun_2019: ['MA']\n",
      "Unique jun_2020: ['MA']\n",
      "Unique jun_2021: ['MA']\n",
      "Unique jul_2017: ['MA']\n",
      "Unique jul_2018: ['MA']\n",
      "Unique jul_2019: ['MA']\n",
      "Unique jul_2020: ['MA']\n",
      "Unique jul_2021: ['MA']\n",
      "Unique aug_2017: ['MA']\n",
      "Unique aug_2018: ['MA']\n",
      "Unique aug_2019: ['MA']\n",
      "Unique aug_2020: ['MA']\n",
      "Unique dec_2017: ['MA']\n",
      "Unique dec_2018: ['MA']\n",
      "Unique dec_2019: ['MA']\n",
      "Unique dec_2020: ['MA']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mMAe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}