DYD_ID-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/ID-day/2017-jan-day-ID.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>efdc20a190efe26df937f8a6e94a6c84d0879b6e</td>\n",
       "      <td>2017-01-01 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>704</td>\n",
       "      <td>704</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>efdc20a190efe26df937f8a6e94a6c84d0879b6e</td>\n",
       "      <td>2017-01-01 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>694</td>\n",
       "      <td>694</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>efdc20a190efe26df937f8a6e94a6c84d0879b6e</td>\n",
       "      <td>2017-01-22 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>249fd610e1444d4f4625b0ad9ec6560d650d14ff</td>\n",
       "      <td>2017-01-11 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>696</td>\n",
       "      <td>712</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11479dda98c732c5c358b8b4b9805adf77cdbad4</td>\n",
       "      <td>2017-01-26 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48304</th>\n",
       "      <td>d97dea968329db6e32e2905331276f2ad5033f22</td>\n",
       "      <td>2017-01-10 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48305</th>\n",
       "      <td>d97dea968329db6e32e2905331276f2ad5033f22</td>\n",
       "      <td>2017-01-10 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48306</th>\n",
       "      <td>d97dea968329db6e32e2905331276f2ad5033f22</td>\n",
       "      <td>2017-01-10 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48307</th>\n",
       "      <td>d97dea968329db6e32e2905331276f2ad5033f22</td>\n",
       "      <td>2017-01-10 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48308</th>\n",
       "      <td>d97dea968329db6e32e2905331276f2ad5033f22</td>\n",
       "      <td>2017-01-10 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>45345 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      efdc20a190efe26df937f8a6e94a6c84d0879b6e  2017-01-01 17:20:00 UTC   \n",
       "1      efdc20a190efe26df937f8a6e94a6c84d0879b6e  2017-01-01 14:05:00 UTC   \n",
       "2      efdc20a190efe26df937f8a6e94a6c84d0879b6e  2017-01-22 19:40:00 UTC   \n",
       "3      249fd610e1444d4f4625b0ad9ec6560d650d14ff  2017-01-11 18:00:00 UTC   \n",
       "4      11479dda98c732c5c358b8b4b9805adf77cdbad4  2017-01-26 19:40:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "48304  d97dea968329db6e32e2905331276f2ad5033f22  2017-01-10 19:05:00 UTC   \n",
       "48305  d97dea968329db6e32e2905331276f2ad5033f22  2017-01-10 18:20:00 UTC   \n",
       "48306  d97dea968329db6e32e2905331276f2ad5033f22  2017-01-10 18:10:00 UTC   \n",
       "48307  d97dea968329db6e32e2905331276f2ad5033f22  2017-01-10 18:35:00 UTC   \n",
       "48308  d97dea968329db6e32e2905331276f2ad5033f22  2017-01-10 18:50:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               703                      704   \n",
       "1         heat          auto               686                      694   \n",
       "2         heat          auto               705                      702   \n",
       "3         heat          hold               678                      696   \n",
       "4         auto          hold               708                      765   \n",
       "...        ...           ...               ...                      ...   \n",
       "48304     heat          auto               758                      760   \n",
       "48305     heat          auto               743                      760   \n",
       "48306     heat          auto               737                      760   \n",
       "48307     heat          auto               752                      760   \n",
       "48308     heat          auto               757                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState        City  Age_of_Home__years_  \\\n",
       "0                          704            ID       Boise                   25   \n",
       "1                          694            ID       Boise                   25   \n",
       "2                          702            ID       Boise                   25   \n",
       "3                          712            ID       Nampa                   10   \n",
       "4                          715            ID       Boise                   40   \n",
       "...                        ...           ...         ...                  ...   \n",
       "48304                      760            ID  Twin Falls                    5   \n",
       "48305                      760            ID  Twin Falls                    5   \n",
       "48306                      760            ID  Twin Falls                    5   \n",
       "48307                      760            ID  Twin Falls                    5   \n",
       "48308                      760            ID  Twin Falls                    5   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  True         False            False   \n",
       "1                  True         False            False   \n",
       "2                  True         False            False   \n",
       "3                  True         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "48304             False         False            False   \n",
       "48305             False         False            False   \n",
       "48306             False         False            False   \n",
       "48307             False         False            False   \n",
       "48308             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "48304                       Gas  \n",
       "48305                       Gas  \n",
       "48306                       Gas  \n",
       "48307                       Gas  \n",
       "48308                       Gas  \n",
       "\n",
       "[45345 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">01e233c4fce6aabdd3f8be666d5f32f972d2a16b</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Eagle</th>\n",
       "      <td>697.392000</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Eagle</th>\n",
       "      <td>706.833333</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>03914e1be819896420f0a91bd59ce372eec7961d</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>twin falls</th>\n",
       "      <td>698.333333</td>\n",
       "      <td>788.333333</td>\n",
       "      <td>708.333333</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>051bb1ef790fd215de91e0304b4f8e06038c2f7d</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>middleton</th>\n",
       "      <td>658.330508</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>060e94e6800d4d62e15d65a113d8f8cb6e0a9983</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Moscow</th>\n",
       "      <td>631.983333</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>85.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f31efa9aa7cd897578fad05d78dedab644e9e923</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Mountain Home</th>\n",
       "      <td>707.433860</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">f461b8107c90a9bf0149592b711a916dbf040b73</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Boise</th>\n",
       "      <td>673.800192</td>\n",
       "      <td>685.043228</td>\n",
       "      <td>676.500480</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Boise</th>\n",
       "      <td>672.926471</td>\n",
       "      <td>675.565126</td>\n",
       "      <td>675.539916</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f789b50593b20342d1df6bba644b219a9ef55b43</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>McCall</th>\n",
       "      <td>683.458333</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>15.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f8c5262ac0960dd4e441eaa7af4447e55326783d</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Boise</th>\n",
       "      <td>676.944444</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                          Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                  \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                    697.392000   \n",
       "                                                             hold          Eagle                    706.833333   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls               698.333333   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                658.330508   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                   631.983333   \n",
       "...                                                                                                        ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home            707.433860   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                    673.800192   \n",
       "                                                             hold          Boise                    672.926471   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                   683.458333   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                    676.944444   \n",
       "\n",
       "                                                                                          TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                           800.000000   \n",
       "                                                             hold          Eagle                           760.000000   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls                      788.333333   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                       700.000000   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                          780.000000   \n",
       "...                                                                                                               ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home                   710.000000   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                           685.043228   \n",
       "                                                             hold          Boise                           675.565126   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                          780.000000   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                           680.000000   \n",
       "\n",
       "                                                                                          TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                           700.000000   \n",
       "                                                             hold          Eagle                           710.000000   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls                      708.333333   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                       660.000000   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                          630.000000   \n",
       "...                                                                                                               ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home                   710.000000   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                           676.500480   \n",
       "                                                             hold          Boise                           675.539916   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                          700.000000   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                           680.000000   \n",
       "\n",
       "                                                                                          Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                         16.0   \n",
       "                                                             hold          Eagle                         16.0   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls                    10.0   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                     10.0   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                        85.0   \n",
       "...                                                                                                       ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home                  5.0   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                         40.0   \n",
       "                                                             hold          Boise                         40.0   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                        15.0   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                          5.0   \n",
       "\n",
       "                                                                                          allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                              \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                     False   \n",
       "                                                             hold          Eagle                     False   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls                False   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                  True   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                    False   \n",
       "...                                                                                                    ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home             False   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                     False   \n",
       "                                                             hold          Boise                     False   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                     True   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                     False   \n",
       "\n",
       "                                                                                          Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                 False   \n",
       "                                                             hold          Eagle                 False   \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls            False   \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton             False   \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                False   \n",
       "...                                                                                                ...   \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home         False   \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                 False   \n",
       "                                                             hold          Boise                 False   \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                False   \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                 False   \n",
       "\n",
       "                                                                                          Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "01e233c4fce6aabdd3f8be666d5f32f972d2a16b Jan   2017 auto     auto          Eagle                    False  \n",
       "                                                             hold          Eagle                    False  \n",
       "03914e1be819896420f0a91bd59ce372eec7961d Jan   2017 heat     auto          twin falls               False  \n",
       "051bb1ef790fd215de91e0304b4f8e06038c2f7d Jan   2017 heat     auto          middleton                False  \n",
       "060e94e6800d4d62e15d65a113d8f8cb6e0a9983 Jan   2017 heat     auto          Moscow                   False  \n",
       "...                                                                                                   ...  \n",
       "f31efa9aa7cd897578fad05d78dedab644e9e923 Jan   2017 heat     hold          Mountain Home            False  \n",
       "f461b8107c90a9bf0149592b711a916dbf040b73 Jan   2017 heat     auto          Boise                    False  \n",
       "                                                             hold          Boise                    False  \n",
       "f789b50593b20342d1df6bba644b219a9ef55b43 Jan   2017 heat     auto          McCall                   False  \n",
       "f8c5262ac0960dd4e441eaa7af4447e55326783d Jan   2017 heat     hold          Boise                    False  \n",
       "\n",
       "[108 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/ID/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/ID-day/2018-jan-day-ID.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ece5a07a140346d57db4e2e4479442e46c86fe1</td>\n",
       "      <td>2018-01-25 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>ID</td>\n",
       "      <td>Chubbuck</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2cfa93b96d6be9fdd4b8735440f26f2ea9b510f9</td>\n",
       "      <td>2018-01-06 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>ID</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c684214553061d899449cdbcf60ea6734f6beb58</td>\n",
       "      <td>2018-01-24 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2018-01-21 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2018-01-24 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>690</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92825</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-01-07 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92826</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-01-20 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92827</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-01-15 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92828</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-01-12 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92829</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-01-06 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>90562 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      9ece5a07a140346d57db4e2e4479442e46c86fe1  2018-01-25 16:10:00 UTC   \n",
       "1      2cfa93b96d6be9fdd4b8735440f26f2ea9b510f9  2018-01-06 17:35:00 UTC   \n",
       "2      c684214553061d899449cdbcf60ea6734f6beb58  2018-01-24 14:00:00 UTC   \n",
       "3      af2e12e051baa0782690f8e4fb33211b8c94ad0a  2018-01-21 16:45:00 UTC   \n",
       "4      af2e12e051baa0782690f8e4fb33211b8c94ad0a  2018-01-24 16:10:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "92825  ea5bc9a01ec9780430554264d1829267d838c012  2018-01-07 14:45:00 UTC   \n",
       "92826  ea5bc9a01ec9780430554264d1829267d838c012  2018-01-20 18:10:00 UTC   \n",
       "92827  ea5bc9a01ec9780430554264d1829267d838c012  2018-01-15 15:25:00 UTC   \n",
       "92828  ea5bc9a01ec9780430554264d1829267d838c012  2018-01-12 19:10:00 UTC   \n",
       "92829  ea5bc9a01ec9780430554264d1829267d838c012  2018-01-06 16:10:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               709                      689   \n",
       "1         heat          hold               664                      675   \n",
       "2         heat          hold               688                      685   \n",
       "3         heat          auto               692                      840   \n",
       "4         heat          auto               690                      840   \n",
       "...        ...           ...               ...                      ...   \n",
       "92825     auto          hold               688                      760   \n",
       "92826     auto          hold               685                      760   \n",
       "92827     auto          hold               684                      760   \n",
       "92828     auto          hold               686                      760   \n",
       "92829     auto          hold               689                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          689            ID  Chubbuck                   15   \n",
       "1                          675            ID     Eagle                   20   \n",
       "2                          685            ID  Meridian                   15   \n",
       "3                          690            ID     eagle                    0   \n",
       "4                          690            ID     eagle                    0   \n",
       "...                        ...           ...       ...                  ...   \n",
       "92825                      690            ID     Boise                   67   \n",
       "92826                      690            ID     Boise                   67   \n",
       "92827                      690            ID     Boise                   67   \n",
       "92828                      690            ID     Boise                   67   \n",
       "92829                      690            ID     Boise                   67   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                  True          True             True   \n",
       "4                  True          True             True   \n",
       "...                 ...           ...              ...   \n",
       "92825             False         False            False   \n",
       "92826             False         False            False   \n",
       "92827             False         False            False   \n",
       "92828             False         False            False   \n",
       "92829             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                      Electric  \n",
       "4                      Electric  \n",
       "...                         ...  \n",
       "92825                       Gas  \n",
       "92826                       Gas  \n",
       "92827                       Gas  \n",
       "92828                       Gas  \n",
       "92829                       Gas  \n",
       "\n",
       "[90562 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/ID/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/ID-day/2019-jan-day-ID.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9ece5a07a140346d57db4e2e4479442e46c86fe1</td>\n",
       "      <td>2019-01-08 08:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>744</td>\n",
       "      <td>694</td>\n",
       "      <td>ID</td>\n",
       "      <td>Chubbuck</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b9b93b8b4437fc59d1edaa609f73580720b1eb3d</td>\n",
       "      <td>2019-01-31 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d'Alene</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a7eb8c843fb19ad3ec1f931b031cec6b2e1b933b</td>\n",
       "      <td>2019-01-28 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>ID</td>\n",
       "      <td>Bellevue</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1231a9bf608fda65f9af0e282dd16577b58bb245</td>\n",
       "      <td>2019-01-20 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>709</td>\n",
       "      <td>714</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>cf97d15e050a6c2af2f2a6bf68d643159ecaacf8</td>\n",
       "      <td>2019-01-27 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>736</td>\n",
       "      <td>736</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159432</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2019-01-26 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159433</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2019-01-12 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159434</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2019-01-12 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159435</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2019-01-12 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159436</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2019-01-12 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>154065 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       9ece5a07a140346d57db4e2e4479442e46c86fe1  2019-01-08 08:20:00 UTC   \n",
       "2       b9b93b8b4437fc59d1edaa609f73580720b1eb3d  2019-01-31 14:55:00 UTC   \n",
       "3       a7eb8c843fb19ad3ec1f931b031cec6b2e1b933b  2019-01-28 14:15:00 UTC   \n",
       "4       1231a9bf608fda65f9af0e282dd16577b58bb245  2019-01-20 13:05:00 UTC   \n",
       "5       cf97d15e050a6c2af2f2a6bf68d643159ecaacf8  2019-01-27 19:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "159432  ea5bc9a01ec9780430554264d1829267d838c012  2019-01-26 16:50:00 UTC   \n",
       "159433  f461b8107c90a9bf0149592b711a916dbf040b73  2019-01-12 19:20:00 UTC   \n",
       "159434  f461b8107c90a9bf0149592b711a916dbf040b73  2019-01-12 19:35:00 UTC   \n",
       "159435  f461b8107c90a9bf0149592b711a916dbf040b73  2019-01-12 17:00:00 UTC   \n",
       "159436  f461b8107c90a9bf0149592b711a916dbf040b73  2019-01-12 18:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          hold               691                      744   \n",
       "2          heat          hold               676                      685   \n",
       "3          heat          hold               654                      655   \n",
       "4          heat          auto               717                      709   \n",
       "5          heat          hold               736                      736   \n",
       "...         ...           ...               ...                      ...   \n",
       "159432     auto          hold               695                      760   \n",
       "159433     auto          auto               711                      760   \n",
       "159434     auto          auto               709                      760   \n",
       "159435     auto          auto               694                      760   \n",
       "159436     auto          auto               709                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           694            ID       Chubbuck   \n",
       "2                           685            ID  Coeur d'Alene   \n",
       "3                           655            ID       Bellevue   \n",
       "4                           714            ID     Twin Falls   \n",
       "5                           736            ID          Boise   \n",
       "...                         ...           ...            ...   \n",
       "159432                      700            ID          Boise   \n",
       "159433                      710            ID          Boise   \n",
       "159434                      710            ID          Boise   \n",
       "159435                      710            ID          Boise   \n",
       "159436                      710            ID          Boise   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        15             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                         5             False         False            False   \n",
       "5                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "159432                   67             False         False            False   \n",
       "159433                   40             False         False            False   \n",
       "159434                   40             False         False            False   \n",
       "159435                   40             False         False            False   \n",
       "159436                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "159432                       Gas  \n",
       "159433                       Gas  \n",
       "159434                       Gas  \n",
       "159435                       Gas  \n",
       "159436                       Gas  \n",
       "\n",
       "[154065 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/ID/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/ID-day/2020-jan-day-ID.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e1031acd23a7535705d7a0689a9ddfc9639b7194</td>\n",
       "      <td>2020-01-10 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>656</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Montpelier</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e1031acd23a7535705d7a0689a9ddfc9639b7194</td>\n",
       "      <td>2020-01-12 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>662</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Montpelier</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e1031acd23a7535705d7a0689a9ddfc9639b7194</td>\n",
       "      <td>2020-01-12 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Montpelier</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e1031acd23a7535705d7a0689a9ddfc9639b7194</td>\n",
       "      <td>2020-01-11 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Montpelier</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e1031acd23a7535705d7a0689a9ddfc9639b7194</td>\n",
       "      <td>2020-01-10 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>656</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Montpelier</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166830</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-01-30 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166831</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-01-30 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166832</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-01-30 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166833</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-01-30 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166834</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-01-30 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>155406 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e1031acd23a7535705d7a0689a9ddfc9639b7194  2020-01-10 18:35:00 UTC   \n",
       "1       e1031acd23a7535705d7a0689a9ddfc9639b7194  2020-01-12 17:25:00 UTC   \n",
       "2       e1031acd23a7535705d7a0689a9ddfc9639b7194  2020-01-12 16:30:00 UTC   \n",
       "3       e1031acd23a7535705d7a0689a9ddfc9639b7194  2020-01-11 18:25:00 UTC   \n",
       "4       e1031acd23a7535705d7a0689a9ddfc9639b7194  2020-01-10 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "166830  f461b8107c90a9bf0149592b711a916dbf040b73  2020-01-30 19:20:00 UTC   \n",
       "166831  f461b8107c90a9bf0149592b711a916dbf040b73  2020-01-30 19:40:00 UTC   \n",
       "166832  f461b8107c90a9bf0149592b711a916dbf040b73  2020-01-30 17:25:00 UTC   \n",
       "166833  f461b8107c90a9bf0149592b711a916dbf040b73  2020-01-30 19:15:00 UTC   \n",
       "166834  f461b8107c90a9bf0149592b711a916dbf040b73  2020-01-30 17:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               656                      660   \n",
       "1          heat          hold               662                      660   \n",
       "2          heat          hold               657                      660   \n",
       "3          heat          hold               654                      660   \n",
       "4          heat          hold               656                      660   \n",
       "...         ...           ...               ...                      ...   \n",
       "166830     auto          hold               708                      760   \n",
       "166831     auto          hold               704                      760   \n",
       "166832     auto          hold               704                      760   \n",
       "166833     auto          hold               704                      760   \n",
       "166834     auto          hold               702                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           660            ID  Montpelier   \n",
       "1                           660            ID  Montpelier   \n",
       "2                           660            ID  Montpelier   \n",
       "3                           660            ID  Montpelier   \n",
       "4                           660            ID  Montpelier   \n",
       "...                         ...           ...         ...   \n",
       "166830                      710            ID       Boise   \n",
       "166831                      710            ID       Boise   \n",
       "166832                      710            ID       Boise   \n",
       "166833                      710            ID       Boise   \n",
       "166834                      710            ID       Boise   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       120              True         False            False   \n",
       "1                       120              True         False            False   \n",
       "2                       120              True         False            False   \n",
       "3                       120              True         False            False   \n",
       "4                       120              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "166830                   40             False         False            False   \n",
       "166831                   40             False         False            False   \n",
       "166832                   40             False         False            False   \n",
       "166833                   40             False         False            False   \n",
       "166834                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "166830                       Gas  \n",
       "166831                       Gas  \n",
       "166832                       Gas  \n",
       "166833                       Gas  \n",
       "166834                       Gas  \n",
       "\n",
       "[155406 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/ID/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/ID-day/2021-jan-day-ID.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14f45a8d01ff96c25e3753b560c3db451cf9ee2f</td>\n",
       "      <td>2021-01-26 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9936ad52f2432f3bccde01a9faaea45239d5de80</td>\n",
       "      <td>2021-01-14 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>774</td>\n",
       "      <td>699</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b8f9a93df6c4871a0c2e37733e8318e930caab47</td>\n",
       "      <td>2021-01-25 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d483926f72d6e83c0253d33cc79762f14612bdeb</td>\n",
       "      <td>2021-01-13 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>824bc89f04deb93327714dc1f08a509089de607b</td>\n",
       "      <td>2021-01-02 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96868</th>\n",
       "      <td>809856e11e16cd2223621fafbdb042533d653a58</td>\n",
       "      <td>2021-01-03 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>730</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96869</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2021-01-07 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96870</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2021-01-07 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96871</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2021-01-07 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96872</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2021-01-07 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92570 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      14f45a8d01ff96c25e3753b560c3db451cf9ee2f  2021-01-26 19:30:00 UTC   \n",
       "1      9936ad52f2432f3bccde01a9faaea45239d5de80  2021-01-14 15:55:00 UTC   \n",
       "2      b8f9a93df6c4871a0c2e37733e8318e930caab47  2021-01-25 19:00:00 UTC   \n",
       "3      d483926f72d6e83c0253d33cc79762f14612bdeb  2021-01-13 14:25:00 UTC   \n",
       "4      824bc89f04deb93327714dc1f08a509089de607b  2021-01-02 18:45:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "96868  809856e11e16cd2223621fafbdb042533d653a58  2021-01-03 19:30:00 UTC   \n",
       "96869  82358b1411142867e615ad6713effeeb472ab863  2021-01-07 15:35:00 UTC   \n",
       "96870  82358b1411142867e615ad6713effeeb472ab863  2021-01-07 15:50:00 UTC   \n",
       "96871  82358b1411142867e615ad6713effeeb472ab863  2021-01-07 15:40:00 UTC   \n",
       "96872  82358b1411142867e615ad6713effeeb472ab863  2021-01-07 15:15:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               753                      755   \n",
       "1         auto          hold               718                      774   \n",
       "2         heat          hold               673                      682   \n",
       "3         heat          hold               678                      675   \n",
       "4         heat          hold               699                      698   \n",
       "...        ...           ...               ...                      ...   \n",
       "96868     auto          hold               729                      760   \n",
       "96869     auto          hold               698                      760   \n",
       "96870     auto          hold               694                      760   \n",
       "96871     auto          hold               697                      760   \n",
       "96872     auto          hold               683                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          755            ID        Nampa   \n",
       "1                          699            ID  Garden City   \n",
       "2                          682            ID     Lewiston   \n",
       "3                          675            ID  Idaho Falls   \n",
       "4                          698            ID        Boise   \n",
       "...                        ...           ...          ...   \n",
       "96868                      730            ID        Boise   \n",
       "96869                      700            ID        Boise   \n",
       "96870                      700            ID        Boise   \n",
       "96871                      700            ID        Boise   \n",
       "96872                      700            ID        Boise   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        5             False         False            False   \n",
       "1                        5             False         False            False   \n",
       "2                        0             False         False            False   \n",
       "3                        5              True         False            False   \n",
       "4                       20              True         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "96868                   20             False         False            False   \n",
       "96869                   30             False         False            False   \n",
       "96870                   30             False         False            False   \n",
       "96871                   30             False         False            False   \n",
       "96872                   30             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "96868                       Gas  \n",
       "96869                       Gas  \n",
       "96870                       Gas  \n",
       "96871                       Gas  \n",
       "96872                       Gas  \n",
       "\n",
       "[92570 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/ID/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>697.392000</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>706.833333</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03914e1be819896420f0a91bd59ce372eec7961d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>twin falls</td>\n",
       "      <td>698.333333</td>\n",
       "      <td>788.333333</td>\n",
       "      <td>708.333333</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>658.330508</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>060e94e6800d4d62e15d65a113d8f8cb6e0a9983</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>631.983333</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>85.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>612.490640</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>614.593604</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>f8c5262ac0960dd4e441eaa7af4447e55326783d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>691.076923</td>\n",
       "      <td>687.384615</td>\n",
       "      <td>669.692308</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>f9e3f96b2929d85dd24eac09985e5d424fcd02de</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>689.916667</td>\n",
       "      <td>690.500000</td>\n",
       "      <td>689.868056</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>695.344262</td>\n",
       "      <td>699.987705</td>\n",
       "      <td>699.987705</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>720.057971</td>\n",
       "      <td>785.801242</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1065 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   Jan  2017     auto   \n",
       "1    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   Jan  2017     auto   \n",
       "2    03914e1be819896420f0a91bd59ce372eec7961d   Jan  2017     heat   \n",
       "3    051bb1ef790fd215de91e0304b4f8e06038c2f7d   Jan  2017     heat   \n",
       "4    060e94e6800d4d62e15d65a113d8f8cb6e0a9983   Jan  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "160  f592e9a2e74bcee311e16c01f7bf7144a899a803   Jan  2021     heat   \n",
       "161  f8c5262ac0960dd4e441eaa7af4447e55326783d   Jan  2021     heat   \n",
       "162  f9e3f96b2929d85dd24eac09985e5d424fcd02de   Jan  2021     heat   \n",
       "163  fa460aad6a17d2a235c05c076ddfc6e429ff15b8   Jan  2021     heat   \n",
       "164  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   Jan  2021     auto   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto        Eagle            697.392000   \n",
       "1            hold        Eagle            706.833333   \n",
       "2            auto   twin falls            698.333333   \n",
       "3            auto    middleton            658.330508   \n",
       "4            auto       Moscow            631.983333   \n",
       "..            ...          ...                   ...   \n",
       "160          hold  Garden City            612.490640   \n",
       "161          hold        Boise            691.076923   \n",
       "162          hold        Boise            689.916667   \n",
       "163          hold        Boise            695.344262   \n",
       "164          hold       Burley            720.057971   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     800.000000                   700.000000   \n",
       "1                     760.000000                   710.000000   \n",
       "2                     788.333333                   708.333333   \n",
       "3                     700.000000                   660.000000   \n",
       "4                     780.000000                   630.000000   \n",
       "..                           ...                          ...   \n",
       "160                   640.000000                   614.593604   \n",
       "161                   687.384615                   669.692308   \n",
       "162                   690.500000                   689.868056   \n",
       "163                   699.987705                   699.987705   \n",
       "164                   785.801242                   730.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   16.0             False         False            False  \n",
       "1                   16.0             False         False            False  \n",
       "2                   10.0             False         False            False  \n",
       "3                   10.0              True         False            False  \n",
       "4                   85.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "160                  0.0             False         False            False  \n",
       "161                  5.0             False         False            False  \n",
       "162                  0.0             False         False            False  \n",
       "163                  5.0             False         False            False  \n",
       "164                 60.0              True         False             True  \n",
       "\n",
       "[1065 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/jan/\" + file)\n",
    "    ID_jan = pd.concat([ID_jan, df])\n",
    "    \n",
    "ID_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_jan.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/ID-day/2017-feb-day-ID.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-02-14 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-02-19 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-02-14 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-02-16 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-02-17 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43489</th>\n",
       "      <td>4cd0c610df7d915c70b1ed8f97a91d85e668cbfe</td>\n",
       "      <td>2017-02-17 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43490</th>\n",
       "      <td>4cd0c610df7d915c70b1ed8f97a91d85e668cbfe</td>\n",
       "      <td>2017-02-16 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43491</th>\n",
       "      <td>4cd0c610df7d915c70b1ed8f97a91d85e668cbfe</td>\n",
       "      <td>2017-02-25 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43492</th>\n",
       "      <td>4cd0c610df7d915c70b1ed8f97a91d85e668cbfe</td>\n",
       "      <td>2017-02-21 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43493</th>\n",
       "      <td>4cd0c610df7d915c70b1ed8f97a91d85e668cbfe</td>\n",
       "      <td>2017-02-17 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>41017 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-02-14 15:45:00 UTC   \n",
       "1      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-02-19 15:15:00 UTC   \n",
       "2      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-02-14 17:30:00 UTC   \n",
       "3      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-02-16 19:40:00 UTC   \n",
       "4      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-02-17 15:35:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "43489  4cd0c610df7d915c70b1ed8f97a91d85e668cbfe  2017-02-17 17:45:00 UTC   \n",
       "43490  4cd0c610df7d915c70b1ed8f97a91d85e668cbfe  2017-02-16 19:05:00 UTC   \n",
       "43491  4cd0c610df7d915c70b1ed8f97a91d85e668cbfe  2017-02-25 18:05:00 UTC   \n",
       "43492  4cd0c610df7d915c70b1ed8f97a91d85e668cbfe  2017-02-21 19:45:00 UTC   \n",
       "43493  4cd0c610df7d915c70b1ed8f97a91d85e668cbfe  2017-02-17 18:35:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               693                      692   \n",
       "1         heat          hold               701                      692   \n",
       "2         heat          hold               688                      692   \n",
       "3         heat          hold               690                      692   \n",
       "4         heat          hold               696                      692   \n",
       "...        ...           ...               ...                      ...   \n",
       "43489     heat          auto               757                      760   \n",
       "43490     heat          auto               758                      760   \n",
       "43491     heat          auto               754                      760   \n",
       "43492     heat          hold               766                      760   \n",
       "43493     heat          auto               757                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState   City  Age_of_Home__years_  \\\n",
       "0                          692            ID  Boise                    5   \n",
       "1                          692            ID  Boise                    5   \n",
       "2                          692            ID  Boise                    5   \n",
       "3                          692            ID  Boise                    5   \n",
       "4                          692            ID  Boise                    5   \n",
       "...                        ...           ...    ...                  ...   \n",
       "43489                      760            ID  Boise                   10   \n",
       "43490                      760            ID  Boise                   10   \n",
       "43491                      760            ID  Boise                   10   \n",
       "43492                      760            ID  Boise                   10   \n",
       "43493                      760            ID  Boise                   10   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "43489             False         False            False   \n",
       "43490             False         False            False   \n",
       "43491             False         False            False   \n",
       "43492             False         False            False   \n",
       "43493             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "43489                       Gas  \n",
       "43490                       Gas  \n",
       "43491                       Gas  \n",
       "43492                       Gas  \n",
       "43493                       Gas  \n",
       "\n",
       "[41017 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/ID/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/ID-day/2018-feb-day-ID.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2018-02-07 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2018-02-21 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2018-02-21 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2018-02-19 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2018-02-14 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88257</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-02-03 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88258</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-02-17 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88259</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-02-05 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88260</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-02-02 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88261</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2018-02-11 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>85790 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      c6965baccac68c3f496346b377af9ae1a6453b6d  2018-02-07 19:55:00 UTC   \n",
       "1      c6965baccac68c3f496346b377af9ae1a6453b6d  2018-02-21 17:30:00 UTC   \n",
       "2      c6965baccac68c3f496346b377af9ae1a6453b6d  2018-02-21 18:50:00 UTC   \n",
       "3      c6965baccac68c3f496346b377af9ae1a6453b6d  2018-02-19 19:20:00 UTC   \n",
       "4      c6965baccac68c3f496346b377af9ae1a6453b6d  2018-02-14 19:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "88257  ea5bc9a01ec9780430554264d1829267d838c012  2018-02-03 19:30:00 UTC   \n",
       "88258  ea5bc9a01ec9780430554264d1829267d838c012  2018-02-17 17:10:00 UTC   \n",
       "88259  ea5bc9a01ec9780430554264d1829267d838c012  2018-02-05 13:50:00 UTC   \n",
       "88260  ea5bc9a01ec9780430554264d1829267d838c012  2018-02-02 19:45:00 UTC   \n",
       "88261  ea5bc9a01ec9780430554264d1829267d838c012  2018-02-11 16:55:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               700                      700   \n",
       "1         heat          hold               704                      700   \n",
       "2         heat          hold               700                      700   \n",
       "3         heat          hold               698                      700   \n",
       "4         heat          hold               703                      700   \n",
       "...        ...           ...               ...                      ...   \n",
       "88257     auto          hold               686                      760   \n",
       "88258     auto          hold               684                      760   \n",
       "88259     auto          hold               686                      760   \n",
       "88260     auto          hold               689                      760   \n",
       "88261     auto          hold               689                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState   City  Age_of_Home__years_  \\\n",
       "0                          700            ID   Star                    5   \n",
       "1                          700            ID   Star                    5   \n",
       "2                          700            ID   Star                    5   \n",
       "3                          700            ID   Star                    5   \n",
       "4                          700            ID   Star                    5   \n",
       "...                        ...           ...    ...                  ...   \n",
       "88257                      690            ID  Boise                   67   \n",
       "88258                      690            ID  Boise                   67   \n",
       "88259                      690            ID  Boise                   67   \n",
       "88260                      690            ID  Boise                   67   \n",
       "88261                      690            ID  Boise                   67   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "88257             False         False            False   \n",
       "88258             False         False            False   \n",
       "88259             False         False            False   \n",
       "88260             False         False            False   \n",
       "88261             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "88257                       Gas  \n",
       "88258                       Gas  \n",
       "88259                       Gas  \n",
       "88260                       Gas  \n",
       "88261                       Gas  \n",
       "\n",
       "[85790 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/ID/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/ID-day/2019-feb-day-ID.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1790c97a1abe8f3ea4655fbbb280ae9bae99cf70</td>\n",
       "      <td>2019-02-15 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>787</td>\n",
       "      <td>727</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2881bbdc8e4014940335c2b156320289d2d0eaf8</td>\n",
       "      <td>2019-02-03 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>766</td>\n",
       "      <td>706</td>\n",
       "      <td>ID</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c5b0593b501324e0a0a35cfd3884ee3bffecfbbe</td>\n",
       "      <td>2019-02-11 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9f68e34e2d006fc39aeea5622ee562c4c23070d8</td>\n",
       "      <td>2019-02-14 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>746</td>\n",
       "      <td>696</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d'Alene</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>9f68e34e2d006fc39aeea5622ee562c4c23070d8</td>\n",
       "      <td>2019-02-20 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>695</td>\n",
       "      <td>746</td>\n",
       "      <td>696</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d'Alene</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113757</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2019-02-28 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113758</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2019-02-23 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113759</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2019-02-23 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113760</th>\n",
       "      <td>ea5bc9a01ec9780430554264d1829267d838c012</td>\n",
       "      <td>2019-02-14 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113761</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2019-02-19 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109414 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1790c97a1abe8f3ea4655fbbb280ae9bae99cf70  2019-02-15 18:30:00 UTC   \n",
       "2       2881bbdc8e4014940335c2b156320289d2d0eaf8  2019-02-03 16:55:00 UTC   \n",
       "3       c5b0593b501324e0a0a35cfd3884ee3bffecfbbe  2019-02-11 14:40:00 UTC   \n",
       "4       9f68e34e2d006fc39aeea5622ee562c4c23070d8  2019-02-14 15:55:00 UTC   \n",
       "5       9f68e34e2d006fc39aeea5622ee562c4c23070d8  2019-02-20 16:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "113757  ea5bc9a01ec9780430554264d1829267d838c012  2019-02-28 14:00:00 UTC   \n",
       "113758  ea5bc9a01ec9780430554264d1829267d838c012  2019-02-23 18:25:00 UTC   \n",
       "113759  ea5bc9a01ec9780430554264d1829267d838c012  2019-02-23 13:40:00 UTC   \n",
       "113760  ea5bc9a01ec9780430554264d1829267d838c012  2019-02-14 19:55:00 UTC   \n",
       "113761  f461b8107c90a9bf0149592b711a916dbf040b73  2019-02-19 19:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               723                      787   \n",
       "2          auto          hold               704                      766   \n",
       "3          auto          hold               706                      755   \n",
       "4          auto          auto               691                      746   \n",
       "5          auto          auto               695                      746   \n",
       "...         ...           ...               ...                      ...   \n",
       "113757     auto          hold               686                      760   \n",
       "113758     auto          hold               688                      760   \n",
       "113759     auto          hold               685                      760   \n",
       "113760     auto          hold               686                      760   \n",
       "113761     auto          auto               696                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           727            ID    Idaho Falls   \n",
       "2                           706            ID          Eagle   \n",
       "3                           705            ID       Lewiston   \n",
       "4                           696            ID  Coeur d'Alene   \n",
       "5                           696            ID  Coeur d'Alene   \n",
       "...                         ...           ...            ...   \n",
       "113757                      690            ID          Boise   \n",
       "113758                      690            ID          Boise   \n",
       "113759                      690            ID          Boise   \n",
       "113760                      690            ID          Boise   \n",
       "113761                      700            ID          Boise   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "2                        20             False         False            False   \n",
       "3                        30              True         False            False   \n",
       "4                        20             False         False            False   \n",
       "5                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "113757                   67             False         False            False   \n",
       "113758                   67             False         False            False   \n",
       "113759                   67             False         False            False   \n",
       "113760                   67             False         False            False   \n",
       "113761                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "113757                       Gas  \n",
       "113758                       Gas  \n",
       "113759                       Gas  \n",
       "113760                       Gas  \n",
       "113761                       Gas  \n",
       "\n",
       "[109414 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/ID/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/ID-day/2020-feb-day-ID.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>46b1845175d7b26c6296f339ba11a9557b248613</td>\n",
       "      <td>2020-02-11 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>ID</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12ca3d29042b0af6a26775df78b8ed4e5da47e8a</td>\n",
       "      <td>2020-02-25 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>712</td>\n",
       "      <td>708</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14f45a8d01ff96c25e3753b560c3db451cf9ee2f</td>\n",
       "      <td>2020-02-01 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>796</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3818b68d3bb9746e331ea025aa36cf68cd83908c</td>\n",
       "      <td>2020-02-01 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>727</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a88da0f39a8f5dcc0827fd65fd814e65183dfd37</td>\n",
       "      <td>2020-02-22 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>806</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Hillcrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155092</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-02-17 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155093</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-02-17 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155094</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-02-17 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155095</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-02-17 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155096</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2020-02-17 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>145364 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       46b1845175d7b26c6296f339ba11a9557b248613  2020-02-11 17:05:00 UTC   \n",
       "2       12ca3d29042b0af6a26775df78b8ed4e5da47e8a  2020-02-25 19:10:00 UTC   \n",
       "3       14f45a8d01ff96c25e3753b560c3db451cf9ee2f  2020-02-01 19:50:00 UTC   \n",
       "4       3818b68d3bb9746e331ea025aa36cf68cd83908c  2020-02-01 19:10:00 UTC   \n",
       "5       a88da0f39a8f5dcc0827fd65fd814e65183dfd37  2020-02-22 15:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "155092  f461b8107c90a9bf0149592b711a916dbf040b73  2020-02-17 19:00:00 UTC   \n",
       "155093  f461b8107c90a9bf0149592b711a916dbf040b73  2020-02-17 16:40:00 UTC   \n",
       "155094  f461b8107c90a9bf0149592b711a916dbf040b73  2020-02-17 17:25:00 UTC   \n",
       "155095  f461b8107c90a9bf0149592b711a916dbf040b73  2020-02-17 17:40:00 UTC   \n",
       "155096  f461b8107c90a9bf0149592b711a916dbf040b73  2020-02-17 19:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               671                      675   \n",
       "2          heat          auto               706                      712   \n",
       "3          heat          auto               757                      796   \n",
       "4          heat          auto               718                      727   \n",
       "5          auto          hold               689                      806   \n",
       "...         ...           ...               ...                      ...   \n",
       "155092     auto          hold               667                      760   \n",
       "155093     auto          hold               674                      760   \n",
       "155094     auto          hold               684                      760   \n",
       "155095     auto          hold               679                      760   \n",
       "155096     auto          hold               664                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           675            ID     Moscow                   50   \n",
       "2                           708            ID   Meridian                    0   \n",
       "3                           760            ID      Nampa                    5   \n",
       "4                           720            ID      Nampa                   15   \n",
       "5                           680            ID  Hillcrest                   40   \n",
       "...                         ...           ...        ...                  ...   \n",
       "155092                      670            ID      Boise                   40   \n",
       "155093                      710            ID      Boise                   40   \n",
       "155094                      670            ID      Boise                   40   \n",
       "155095                      670            ID      Boise                   40   \n",
       "155096                      670            ID      Boise                   40   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "2                   True         False             True   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "5                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "155092             False         False            False   \n",
       "155093             False         False            False   \n",
       "155094             False         False            False   \n",
       "155095             False         False            False   \n",
       "155096             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "155092                       Gas  \n",
       "155093                       Gas  \n",
       "155094                       Gas  \n",
       "155095                       Gas  \n",
       "155096                       Gas  \n",
       "\n",
       "[145364 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/ID/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/ID-day/2021-feb-day-ID.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0a36c8882151c1e485ff07cdbda65bd3bb0a8e17</td>\n",
       "      <td>2021-02-07 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f2fbf8989c48ef7f73662ee36e35ff771cb37bc2</td>\n",
       "      <td>2021-02-07 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b8f9a93df6c4871a0c2e37733e8318e930caab47</td>\n",
       "      <td>2021-02-18 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>672</td>\n",
       "      <td>672</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0a36c8882151c1e485ff07cdbda65bd3bb0a8e17</td>\n",
       "      <td>2021-02-26 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a3383e4e6d75718876a3fe1fcb7dbe788f4e03e8</td>\n",
       "      <td>2021-02-02 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>757</td>\n",
       "      <td>677</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86865</th>\n",
       "      <td>adfc25f436aac9d0b3d81b774bca297c8895eb03</td>\n",
       "      <td>2021-02-26 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86866</th>\n",
       "      <td>adfc25f436aac9d0b3d81b774bca297c8895eb03</td>\n",
       "      <td>2021-02-26 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86867</th>\n",
       "      <td>adfc25f436aac9d0b3d81b774bca297c8895eb03</td>\n",
       "      <td>2021-02-26 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86868</th>\n",
       "      <td>adfc25f436aac9d0b3d81b774bca297c8895eb03</td>\n",
       "      <td>2021-02-26 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86869</th>\n",
       "      <td>adfc25f436aac9d0b3d81b774bca297c8895eb03</td>\n",
       "      <td>2021-02-26 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80855 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      0a36c8882151c1e485ff07cdbda65bd3bb0a8e17  2021-02-07 17:55:00 UTC   \n",
       "1      f2fbf8989c48ef7f73662ee36e35ff771cb37bc2  2021-02-07 18:00:00 UTC   \n",
       "2      b8f9a93df6c4871a0c2e37733e8318e930caab47  2021-02-18 18:20:00 UTC   \n",
       "3      0a36c8882151c1e485ff07cdbda65bd3bb0a8e17  2021-02-26 14:55:00 UTC   \n",
       "4      a3383e4e6d75718876a3fe1fcb7dbe788f4e03e8  2021-02-02 17:05:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "86865  adfc25f436aac9d0b3d81b774bca297c8895eb03  2021-02-26 17:55:00 UTC   \n",
       "86866  adfc25f436aac9d0b3d81b774bca297c8895eb03  2021-02-26 19:05:00 UTC   \n",
       "86867  adfc25f436aac9d0b3d81b774bca297c8895eb03  2021-02-26 17:50:00 UTC   \n",
       "86868  adfc25f436aac9d0b3d81b774bca297c8895eb03  2021-02-26 15:30:00 UTC   \n",
       "86869  adfc25f436aac9d0b3d81b774bca297c8895eb03  2021-02-26 17:05:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               676                      665   \n",
       "1         heat          hold               700                      705   \n",
       "2         heat          hold               683                      672   \n",
       "3         heat          hold               679                      685   \n",
       "4         auto          hold               676                      757   \n",
       "...        ...           ...               ...                      ...   \n",
       "86865     auto          hold               697                      760   \n",
       "86866     auto          hold               690                      760   \n",
       "86867     auto          hold               695                      760   \n",
       "86868     auto          hold               696                      760   \n",
       "86869     auto          hold               692                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          665            ID     Boise                   59   \n",
       "1                          705            ID  Meridian                    5   \n",
       "2                          672            ID  Lewiston                    0   \n",
       "3                          685            ID     Boise                   59   \n",
       "4                          677            ID      Kuna                   15   \n",
       "...                        ...           ...       ...                  ...   \n",
       "86865                      700            ID      Kuna                   20   \n",
       "86866                      700            ID      Kuna                   20   \n",
       "86867                      700            ID      Kuna                   20   \n",
       "86868                      700            ID      Kuna                   20   \n",
       "86869                      700            ID      Kuna                   20   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False             True   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False             True   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "86865             False         False            False   \n",
       "86866             False         False            False   \n",
       "86867             False         False            False   \n",
       "86868             False         False            False   \n",
       "86869             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                      Electric  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                      Electric  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "86865                       Gas  \n",
       "86866                       Gas  \n",
       "86867                       Gas  \n",
       "86868                       Gas  \n",
       "86869                       Gas  \n",
       "\n",
       "[80855 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/ID/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>665.659524</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>671.685714</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>060e94e6800d4d62e15d65a113d8f8cb6e0a9983</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>632.500000</td>\n",
       "      <td>666.166667</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>85.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>060e94e6800d4d62e15d65a113d8f8cb6e0a9983</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>644.000000</td>\n",
       "      <td>655.388889</td>\n",
       "      <td>649.166667</td>\n",
       "      <td>85.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Firth</td>\n",
       "      <td>679.500000</td>\n",
       "      <td>817.500000</td>\n",
       "      <td>669.500000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Firth</td>\n",
       "      <td>694.641975</td>\n",
       "      <td>686.728395</td>\n",
       "      <td>686.358025</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>f8c5262ac0960dd4e441eaa7af4447e55326783d</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>685.555556</td>\n",
       "      <td>693.444444</td>\n",
       "      <td>685.666667</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>f9e3f96b2929d85dd24eac09985e5d424fcd02de</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>694.732591</td>\n",
       "      <td>695.537604</td>\n",
       "      <td>695.222841</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>699.842334</td>\n",
       "      <td>704.743031</td>\n",
       "      <td>704.743031</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>696.666667</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>720.262363</td>\n",
       "      <td>790.357143</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1006 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    051bb1ef790fd215de91e0304b4f8e06038c2f7d   feb  2017     heat   \n",
       "1    060e94e6800d4d62e15d65a113d8f8cb6e0a9983   feb  2017     heat   \n",
       "2    060e94e6800d4d62e15d65a113d8f8cb6e0a9983   feb  2017     heat   \n",
       "3    08e17f959d8fff4ab5f1bab6028a513900f97800   feb  2017     heat   \n",
       "4    08e17f959d8fff4ab5f1bab6028a513900f97800   feb  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "151  f8c5262ac0960dd4e441eaa7af4447e55326783d   feb  2021     heat   \n",
       "152  f9e3f96b2929d85dd24eac09985e5d424fcd02de   feb  2021     heat   \n",
       "153  fa460aad6a17d2a235c05c076ddfc6e429ff15b8   feb  2021     heat   \n",
       "154  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   feb  2021     heat   \n",
       "155  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   feb  2021     auto   \n",
       "\n",
       "    CalendarEvent       City  Temperature_ctrl_ave  \\\n",
       "0            auto  middleton            665.659524   \n",
       "1            auto     Moscow            632.500000   \n",
       "2            hold     Moscow            644.000000   \n",
       "3            auto      Firth            679.500000   \n",
       "4            hold      Firth            694.641975   \n",
       "..            ...        ...                   ...   \n",
       "151          hold      Boise            685.555556   \n",
       "152          hold      Boise            694.732591   \n",
       "153          hold      Boise            699.842334   \n",
       "154          hold       Kuna            696.666667   \n",
       "155          hold     Burley            720.262363   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     700.000000                   671.685714   \n",
       "1                     666.166667                   630.000000   \n",
       "2                     655.388889                   649.166667   \n",
       "3                     817.500000                   669.500000   \n",
       "4                     686.728395                   686.358025   \n",
       "..                           ...                          ...   \n",
       "151                   693.444444                   685.666667   \n",
       "152                   695.537604                   695.222841   \n",
       "153                   704.743031                   704.743031   \n",
       "154                   670.000000                   670.000000   \n",
       "155                   790.357143                   730.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   10.0              True         False            False  \n",
       "1                   85.0             False         False            False  \n",
       "2                   85.0             False         False            False  \n",
       "3                   50.0             False         False            False  \n",
       "4                   50.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "151                  5.0             False         False            False  \n",
       "152                  0.0             False         False            False  \n",
       "153                  5.0             False         False            False  \n",
       "154                  0.0             False         False            False  \n",
       "155                 60.0              True         False             True  \n",
       "\n",
       "[1006 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/feb/\" + file)\n",
    "    ID_feb = pd.concat([ID_feb, df])\n",
    "    \n",
    "ID_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_feb.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/ID-day/2017-jun-day-ID.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0eb9f31787410572e6cb0995b6af77ae6f13dacc</td>\n",
       "      <td>2017-06-25 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>675</td>\n",
       "      <td>625</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0eb9f31787410572e6cb0995b6af77ae6f13dacc</td>\n",
       "      <td>2017-06-07 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>665</td>\n",
       "      <td>615</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0eb9f31787410572e6cb0995b6af77ae6f13dacc</td>\n",
       "      <td>2017-06-30 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-06-27 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>732</td>\n",
       "      <td>732</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e62933d8740a1a22883ff769a19f0f69d2e6c974</td>\n",
       "      <td>2017-06-06 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>605</td>\n",
       "      <td>ID</td>\n",
       "      <td>Blackfoot</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74356</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-06-28 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74357</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-06-20 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>770</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74358</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-06-20 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>769</td>\n",
       "      <td>760</td>\n",
       "      <td>770</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74359</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-06-28 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74360</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-06-28 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>72102 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      0eb9f31787410572e6cb0995b6af77ae6f13dacc  2017-06-25 12:35:00 UTC   \n",
       "1      0eb9f31787410572e6cb0995b6af77ae6f13dacc  2017-06-07 12:55:00 UTC   \n",
       "2      0eb9f31787410572e6cb0995b6af77ae6f13dacc  2017-06-30 12:45:00 UTC   \n",
       "3      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-06-27 16:25:00 UTC   \n",
       "4      e62933d8740a1a22883ff769a19f0f69d2e6c974  2017-06-06 18:55:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "74356  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-06-28 18:00:00 UTC   \n",
       "74357  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-06-20 14:00:00 UTC   \n",
       "74358  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-06-20 17:00:00 UTC   \n",
       "74359  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-06-28 15:55:00 UTC   \n",
       "74360  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-06-28 17:50:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         auto          hold               677                      675   \n",
       "1         auto          hold               666                      665   \n",
       "2         cool          hold               674                      675   \n",
       "3         cool          hold               722                      732   \n",
       "4         auto          hold               705                      705   \n",
       "...        ...           ...               ...                      ...   \n",
       "74356     cool          auto               763                      760   \n",
       "74357     cool          auto               758                      760   \n",
       "74358     cool          auto               769                      760   \n",
       "74359     cool          auto               763                      760   \n",
       "74360     cool          auto               762                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                          625            ID          Boise   \n",
       "1                          615            ID          Boise   \n",
       "2                          675            ID          Boise   \n",
       "3                          732            ID          Boise   \n",
       "4                          605            ID      Blackfoot   \n",
       "...                        ...           ...            ...   \n",
       "74356                      790            ID  Mountain Home   \n",
       "74357                      770            ID  Mountain Home   \n",
       "74358                      770            ID  Mountain Home   \n",
       "74359                      790            ID  Mountain Home   \n",
       "74360                      790            ID  Mountain Home   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        0             False         False            False   \n",
       "1                        0             False         False            False   \n",
       "2                        0             False         False            False   \n",
       "3                        5             False         False            False   \n",
       "4                       20             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "74356                    5             False         False            False   \n",
       "74357                    5             False         False            False   \n",
       "74358                    5             False         False            False   \n",
       "74359                    5             False         False            False   \n",
       "74360                    5             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "74356                       Gas  \n",
       "74357                       Gas  \n",
       "74358                       Gas  \n",
       "74359                       Gas  \n",
       "74360                       Gas  \n",
       "\n",
       "[72102 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/ID/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/ID-day/2018-jun-day-ID.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21</td>\n",
       "      <td>2018-06-20 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>717</td>\n",
       "      <td>717</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e62933d8740a1a22883ff769a19f0f69d2e6c974</td>\n",
       "      <td>2018-06-26 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Blackfoot</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2018-06-29 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>840</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e62933d8740a1a22883ff769a19f0f69d2e6c974</td>\n",
       "      <td>2018-06-20 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Blackfoot</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7b8a7293df330c37984c20f9924ab8edfe0141dc</td>\n",
       "      <td>2018-06-11 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>712</td>\n",
       "      <td>662</td>\n",
       "      <td>ID</td>\n",
       "      <td>Soda Springs</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114270</th>\n",
       "      <td>e34b2144b0c193899a53fbd27a3ae38e1e026832</td>\n",
       "      <td>2018-06-05 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114283</th>\n",
       "      <td>e34b2144b0c193899a53fbd27a3ae38e1e026832</td>\n",
       "      <td>2018-06-05 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114295</th>\n",
       "      <td>e34b2144b0c193899a53fbd27a3ae38e1e026832</td>\n",
       "      <td>2018-06-05 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114301</th>\n",
       "      <td>e34b2144b0c193899a53fbd27a3ae38e1e026832</td>\n",
       "      <td>2018-06-05 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114319</th>\n",
       "      <td>e34b2144b0c193899a53fbd27a3ae38e1e026832</td>\n",
       "      <td>2018-06-05 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>106416 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21  2018-06-20 15:30:00 UTC   \n",
       "1       e62933d8740a1a22883ff769a19f0f69d2e6c974  2018-06-26 17:20:00 UTC   \n",
       "2       af2e12e051baa0782690f8e4fb33211b8c94ad0a  2018-06-29 15:20:00 UTC   \n",
       "3       e62933d8740a1a22883ff769a19f0f69d2e6c974  2018-06-20 18:35:00 UTC   \n",
       "4       7b8a7293df330c37984c20f9924ab8edfe0141dc  2018-06-11 18:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "114270  e34b2144b0c193899a53fbd27a3ae38e1e026832  2018-06-05 15:10:00 UTC   \n",
       "114283  e34b2144b0c193899a53fbd27a3ae38e1e026832  2018-06-05 14:35:00 UTC   \n",
       "114295  e34b2144b0c193899a53fbd27a3ae38e1e026832  2018-06-05 14:25:00 UTC   \n",
       "114301  e34b2144b0c193899a53fbd27a3ae38e1e026832  2018-06-05 13:35:00 UTC   \n",
       "114319  e34b2144b0c193899a53fbd27a3ae38e1e026832  2018-06-05 16:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               721                      717   \n",
       "1          cool          hold               686                      685   \n",
       "2          cool          auto               707                      840   \n",
       "3          cool          hold               689                      685   \n",
       "4          auto          hold               661                      712   \n",
       "...         ...           ...               ...                      ...   \n",
       "114270     auto          hold               688                      760   \n",
       "114283     auto          hold               685                      760   \n",
       "114295     auto          hold               686                      760   \n",
       "114301     auto          hold               689                      760   \n",
       "114319     auto          hold               689                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           717            ID    Twin Falls   \n",
       "1                           685            ID     Blackfoot   \n",
       "2                           680            ID         eagle   \n",
       "3                           685            ID     Blackfoot   \n",
       "4                           662            ID  Soda Springs   \n",
       "...                         ...           ...           ...   \n",
       "114270                      630            ID         Boise   \n",
       "114283                      630            ID         Boise   \n",
       "114295                      630            ID         Boise   \n",
       "114301                      630            ID         Boise   \n",
       "114319                      630            ID         Boise   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                         0              True          True             True   \n",
       "3                        20             False         False            False   \n",
       "4                        80             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "114270                   25             False         False            False   \n",
       "114283                   25             False         False            False   \n",
       "114295                   25             False         False            False   \n",
       "114301                   25             False         False            False   \n",
       "114319                   25             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "114270                       Gas  \n",
       "114283                       Gas  \n",
       "114295                       Gas  \n",
       "114301                       Gas  \n",
       "114319                       Gas  \n",
       "\n",
       "[106416 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/ID/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/ID-day/2019-jun-day-ID.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2881bbdc8e4014940335c2b156320289d2d0eaf8</td>\n",
       "      <td>2019-06-17 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>ID</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ed456d483408d5b0cbe21a04e24a1b05d5a23976</td>\n",
       "      <td>2019-06-02 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d Alene</td>\n",
       "      <td>7</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3818b68d3bb9746e331ea025aa36cf68cd83908c</td>\n",
       "      <td>2019-06-18 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>2019-06-06 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>712</td>\n",
       "      <td>662</td>\n",
       "      <td>ID</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2bc39b4d34e56e308a9c028cdb1955780546a0f6</td>\n",
       "      <td>2019-06-10 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188839</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2019-06-22 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188840</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2019-06-01 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188841</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2019-06-22 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188842</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2019-06-22 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188843</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2019-06-22 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>178386 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2881bbdc8e4014940335c2b156320289d2d0eaf8  2019-06-17 17:55:00 UTC   \n",
       "1       ed456d483408d5b0cbe21a04e24a1b05d5a23976  2019-06-02 18:50:00 UTC   \n",
       "2       3818b68d3bb9746e331ea025aa36cf68cd83908c  2019-06-18 16:05:00 UTC   \n",
       "3       fa9e861361c9be63c34bdb1d26f4718da1e69d4f  2019-06-06 18:40:00 UTC   \n",
       "4       2bc39b4d34e56e308a9c028cdb1955780546a0f6  2019-06-10 14:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "188839  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2019-06-22 17:30:00 UTC   \n",
       "188840  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2019-06-01 12:45:00 UTC   \n",
       "188841  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2019-06-22 18:55:00 UTC   \n",
       "188842  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2019-06-22 17:10:00 UTC   \n",
       "188843  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2019-06-22 14:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               718                      716   \n",
       "1          auto          hold               728                      727   \n",
       "2          cool          hold               702                      697   \n",
       "3          cool          auto               717                      712   \n",
       "4          auto          hold               729                      785   \n",
       "...         ...           ...               ...                      ...   \n",
       "188839     auto          hold               729                      760   \n",
       "188840     auto          auto               739                      760   \n",
       "188841     auto          hold               729                      760   \n",
       "188842     auto          hold               727                      760   \n",
       "188843     auto          hold               724                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           716            ID          Eagle   \n",
       "1                           677            ID  Coeur d Alene   \n",
       "2                           697            ID          Nampa   \n",
       "3                           662            ID           Kuna   \n",
       "4                           735            ID          Boise   \n",
       "...                         ...           ...            ...   \n",
       "188839                      720            ID         Burley   \n",
       "188840                      720            ID         Burley   \n",
       "188841                      720            ID         Burley   \n",
       "188842                      720            ID         Burley   \n",
       "188843                      720            ID         Burley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                         7              True         False            False   \n",
       "2                        15             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "188839                   60              True         False             True   \n",
       "188840                   60              True         False             True   \n",
       "188841                   60              True         False             True   \n",
       "188842                   60              True         False             True   \n",
       "188843                   60              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "188839                  Electric  \n",
       "188840                  Electric  \n",
       "188841                  Electric  \n",
       "188842                  Electric  \n",
       "188843                  Electric  \n",
       "\n",
       "[178386 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/ID/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/ID-day/2020-jun-day-ID.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>253543009c667cbcfd69450d5405908cb4901372</td>\n",
       "      <td>2020-06-22 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>726</td>\n",
       "      <td>676</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e6eb197f10740f7d58d52b8cbb59b54bc7239aa7</td>\n",
       "      <td>2020-06-21 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>90dcbe67f8b9c2bf761a9ef2f5e6c0a999f4d211</td>\n",
       "      <td>2020-06-24 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>755</td>\n",
       "      <td>695</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>253543009c667cbcfd69450d5405908cb4901372</td>\n",
       "      <td>2020-06-21 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>726</td>\n",
       "      <td>676</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e6eb197f10740f7d58d52b8cbb59b54bc7239aa7</td>\n",
       "      <td>2020-06-22 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>706</td>\n",
       "      <td>706</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178286</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-06-16 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178287</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-06-15 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178288</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-06-16 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178289</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-06-01 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178290</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-06-16 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>171911 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       253543009c667cbcfd69450d5405908cb4901372  2020-06-22 14:55:00 UTC   \n",
       "1       e6eb197f10740f7d58d52b8cbb59b54bc7239aa7  2020-06-21 17:50:00 UTC   \n",
       "2       90dcbe67f8b9c2bf761a9ef2f5e6c0a999f4d211  2020-06-24 15:40:00 UTC   \n",
       "3       253543009c667cbcfd69450d5405908cb4901372  2020-06-21 19:15:00 UTC   \n",
       "4       e6eb197f10740f7d58d52b8cbb59b54bc7239aa7  2020-06-22 13:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "178286  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-06-16 17:35:00 UTC   \n",
       "178287  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-06-15 14:20:00 UTC   \n",
       "178288  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-06-16 14:20:00 UTC   \n",
       "178289  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-06-01 12:50:00 UTC   \n",
       "178290  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-06-16 18:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               716                      726   \n",
       "1          cool          hold               707                      706   \n",
       "2          auto          hold               744                      755   \n",
       "3          auto          hold               731                      726   \n",
       "4          cool          hold               668                      706   \n",
       "...         ...           ...               ...                      ...   \n",
       "178286     auto          auto               725                      760   \n",
       "178287     auto          auto               706                      760   \n",
       "178288     auto          auto               719                      760   \n",
       "178289     auto          hold               734                      760   \n",
       "178290     auto          auto               725                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           676            ID     Boise                    0   \n",
       "1                           706            ID  Meridian                   19   \n",
       "2                           695            ID     Boise                   20   \n",
       "3                           676            ID     Boise                    0   \n",
       "4                           706            ID  Meridian                   19   \n",
       "...                         ...           ...       ...                  ...   \n",
       "178286                      720            ID    Burley                   60   \n",
       "178287                      720            ID    Burley                   60   \n",
       "178288                      720            ID    Burley                   60   \n",
       "178289                      700            ID    Burley                   60   \n",
       "178290                      720            ID    Burley                   60   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                   True         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                   True         False            False   \n",
       "...                  ...           ...              ...   \n",
       "178286              True         False             True   \n",
       "178287              True         False             True   \n",
       "178288              True         False             True   \n",
       "178289              True         False             True   \n",
       "178290              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "178286                  Electric  \n",
       "178287                  Electric  \n",
       "178288                  Electric  \n",
       "178289                  Electric  \n",
       "178290                  Electric  \n",
       "\n",
       "[171911 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/ID/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/ID-day/2021-jun-day-ID.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b8f9a93df6c4871a0c2e37733e8318e930caab47</td>\n",
       "      <td>2021-06-08 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>682</td>\n",
       "      <td>682</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ba0af571094f2f350671321308ffc666ec675d0d</td>\n",
       "      <td>2021-06-25 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>729</td>\n",
       "      <td>718</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b8f9a93df6c4871a0c2e37733e8318e930caab47</td>\n",
       "      <td>2021-06-29 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>672</td>\n",
       "      <td>672</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b8f9a93df6c4871a0c2e37733e8318e930caab47</td>\n",
       "      <td>2021-06-27 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>692</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>Lewiston</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>253543009c667cbcfd69450d5405908cb4901372</td>\n",
       "      <td>2021-06-03 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>718</td>\n",
       "      <td>668</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117168</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2021-06-03 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117169</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2021-06-04 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117170</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2021-06-05 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117171</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2021-06-06 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117172</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2021-06-03 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108892 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b8f9a93df6c4871a0c2e37733e8318e930caab47  2021-06-08 15:40:00 UTC   \n",
       "1       ba0af571094f2f350671321308ffc666ec675d0d  2021-06-25 14:05:00 UTC   \n",
       "2       b8f9a93df6c4871a0c2e37733e8318e930caab47  2021-06-29 17:45:00 UTC   \n",
       "3       b8f9a93df6c4871a0c2e37733e8318e930caab47  2021-06-27 16:40:00 UTC   \n",
       "4       253543009c667cbcfd69450d5405908cb4901372  2021-06-03 18:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "117168  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2021-06-03 13:40:00 UTC   \n",
       "117169  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2021-06-04 17:50:00 UTC   \n",
       "117170  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2021-06-05 17:25:00 UTC   \n",
       "117171  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2021-06-06 17:25:00 UTC   \n",
       "117172  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2021-06-03 17:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               699                      682   \n",
       "1          cool          hold               727                      729   \n",
       "2          cool          hold               683                      672   \n",
       "3          cool          hold               700                      692   \n",
       "4          auto          hold               713                      718   \n",
       "...         ...           ...               ...                      ...   \n",
       "117168     auto          hold               751                      760   \n",
       "117169     auto          hold               760                      760   \n",
       "117170     auto          hold               755                      760   \n",
       "117171     auto          hold               728                      760   \n",
       "117172     auto          hold               758                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           682            ID  Lewiston                    0   \n",
       "1                           718            ID     Boise                   60   \n",
       "2                           672            ID  Lewiston                    0   \n",
       "3                           692            ID  Lewiston                    0   \n",
       "4                           668            ID     Boise                    0   \n",
       "...                         ...           ...       ...                  ...   \n",
       "117168                      700            ID    Burley                   60   \n",
       "117169                      700            ID    Burley                   60   \n",
       "117170                      700            ID    Burley                   60   \n",
       "117171                      700            ID    Burley                   60   \n",
       "117172                      700            ID    Burley                   60   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                   True         False             True   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "117168              True         False             True   \n",
       "117169              True         False             True   \n",
       "117170              True         False             True   \n",
       "117171              True         False             True   \n",
       "117172              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "117168                  Electric  \n",
       "117169                  Electric  \n",
       "117170                  Electric  \n",
       "117171                  Electric  \n",
       "117172                  Electric  \n",
       "\n",
       "[108892 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/ID/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>738.204082</td>\n",
       "      <td>725.346939</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>731.179487</td>\n",
       "      <td>724.871795</td>\n",
       "      <td>693.782051</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>698.273453</td>\n",
       "      <td>699.996008</td>\n",
       "      <td>660.003992</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>middleton</td>\n",
       "      <td>704.666667</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Firth</td>\n",
       "      <td>754.111111</td>\n",
       "      <td>760.055556</td>\n",
       "      <td>640.388889</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>f9e3f96b2929d85dd24eac09985e5d424fcd02de</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>735.618312</td>\n",
       "      <td>738.469939</td>\n",
       "      <td>738.399890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>709.340206</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>697.672269</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>734.242340</td>\n",
       "      <td>746.652925</td>\n",
       "      <td>699.992201</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>ff576006ddec2339bd9c9bcbef140daeba69bae1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>735.821429</td>\n",
       "      <td>740.571429</td>\n",
       "      <td>739.571429</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1348 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   jun  2017     cool   \n",
       "1    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   jun  2017     cool   \n",
       "2    051bb1ef790fd215de91e0304b4f8e06038c2f7d   jun  2017     cool   \n",
       "3    051bb1ef790fd215de91e0304b4f8e06038c2f7d   jun  2017     cool   \n",
       "4    08e17f959d8fff4ab5f1bab6028a513900f97800   jun  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "220  f9e3f96b2929d85dd24eac09985e5d424fcd02de   jun  2021     cool   \n",
       "221  fa460aad6a17d2a235c05c076ddfc6e429ff15b8   jun  2021     auto   \n",
       "222  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   jun  2021     cool   \n",
       "223  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   jun  2021     auto   \n",
       "224  ff576006ddec2339bd9c9bcbef140daeba69bae1   jun  2021     cool   \n",
       "\n",
       "    CalendarEvent       City  Temperature_ctrl_ave  \\\n",
       "0            auto      Eagle            738.204082   \n",
       "1            hold      Eagle            731.179487   \n",
       "2            auto  middleton            698.273453   \n",
       "3            hold  middleton            704.666667   \n",
       "4            auto      Firth            754.111111   \n",
       "..            ...        ...                   ...   \n",
       "220          hold      Boise            735.618312   \n",
       "221          hold      Boise            709.340206   \n",
       "222          hold       Kuna            697.672269   \n",
       "223          hold     Burley            734.242340   \n",
       "224          hold   Meridian            735.821429   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     725.346939                   690.000000   \n",
       "1                     724.871795                   693.782051   \n",
       "2                     699.996008                   660.003992   \n",
       "3                     680.000000                   680.000000   \n",
       "4                     760.055556                   640.388889   \n",
       "..                           ...                          ...   \n",
       "220                   738.469939                   738.399890   \n",
       "221                   750.000000                   680.000000   \n",
       "222                   690.000000                   690.000000   \n",
       "223                   746.652925                   699.992201   \n",
       "224                   740.571429                   739.571429   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   16.0             False         False            False  \n",
       "1                   16.0             False         False            False  \n",
       "2                   10.0              True         False            False  \n",
       "3                   10.0              True         False            False  \n",
       "4                   50.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "220                  0.0             False         False            False  \n",
       "221                  5.0             False         False            False  \n",
       "222                  0.0             False         False            False  \n",
       "223                 60.0              True         False             True  \n",
       "224                 17.0             False         False            False  \n",
       "\n",
       "[1348 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/jun/\" + file)\n",
    "    ID_jun = pd.concat([ID_jun, df])\n",
    "    \n",
    "ID_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_jun.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/ID-day/2017-jul-day-ID.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cf97d15e050a6c2af2f2a6bf68d643159ecaacf8</td>\n",
       "      <td>2017-07-07 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2017-07-25 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>790</td>\n",
       "      <td>810</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d3c131ce6f50b46e0c93ac3dbe8f6244e3c51875</td>\n",
       "      <td>2017-07-30 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>712</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>twin falls</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6ef2a624b0bd1403255f7e2ff28a54084bb73cd1</td>\n",
       "      <td>2017-07-07 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>731</td>\n",
       "      <td>677</td>\n",
       "      <td>ID</td>\n",
       "      <td>meridian</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1790c97a1abe8f3ea4655fbbb280ae9bae99cf70</td>\n",
       "      <td>2017-07-08 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>752</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98402</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-07-09 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98403</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-07-09 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98404</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-07-09 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98405</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-07-09 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98406</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-07-09 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>740</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97696 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      cf97d15e050a6c2af2f2a6bf68d643159ecaacf8  2017-07-07 19:25:00 UTC   \n",
       "1      af2e12e051baa0782690f8e4fb33211b8c94ad0a  2017-07-25 19:25:00 UTC   \n",
       "2      d3c131ce6f50b46e0c93ac3dbe8f6244e3c51875  2017-07-30 18:30:00 UTC   \n",
       "3      6ef2a624b0bd1403255f7e2ff28a54084bb73cd1  2017-07-07 12:25:00 UTC   \n",
       "4      1790c97a1abe8f3ea4655fbbb280ae9bae99cf70  2017-07-08 17:25:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "98402  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-07-09 18:35:00 UTC   \n",
       "98403  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-07-09 17:55:00 UTC   \n",
       "98404  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-07-09 14:15:00 UTC   \n",
       "98405  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-07-09 18:25:00 UTC   \n",
       "98406  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-07-09 15:45:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          hold               748                      742   \n",
       "1         cool          hold               790                      810   \n",
       "2         cool          auto               706                      712   \n",
       "3         auto          auto               725                      731   \n",
       "4         cool          auto               758                      752   \n",
       "...        ...           ...               ...                      ...   \n",
       "98402     cool          auto               764                      760   \n",
       "98403     cool          auto               760                      760   \n",
       "98404     cool          auto               753                      760   \n",
       "98405     cool          auto               765                      760   \n",
       "98406     cool          auto               762                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                          742            ID          Boise   \n",
       "1                          740            ID          eagle   \n",
       "2                          690            ID     twin falls   \n",
       "3                          677            ID       meridian   \n",
       "4                          690            ID    Idaho Falls   \n",
       "...                        ...           ...            ...   \n",
       "98402                      740            ID  Mountain Home   \n",
       "98403                      740            ID  Mountain Home   \n",
       "98404                      740            ID  Mountain Home   \n",
       "98405                      740            ID  Mountain Home   \n",
       "98406                      740            ID  Mountain Home   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       20             False         False            False   \n",
       "1                        0              True          True             True   \n",
       "2                       10              True         False            False   \n",
       "3                        5             False         False            False   \n",
       "4                       10             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "98402                    5             False         False            False   \n",
       "98403                    5             False         False            False   \n",
       "98404                    5             False         False            False   \n",
       "98405                    5             False         False            False   \n",
       "98406                    5             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                      Electric  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "98402                       Gas  \n",
       "98403                       Gas  \n",
       "98404                       Gas  \n",
       "98405                       Gas  \n",
       "98406                       Gas  \n",
       "\n",
       "[97696 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/ID/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/ID-day/2018-jul-day-ID.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9ece5a07a140346d57db4e2e4479442e46c86fe1</td>\n",
       "      <td>2018-07-20 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>754</td>\n",
       "      <td>665</td>\n",
       "      <td>ID</td>\n",
       "      <td>Chubbuck</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9d62ba3a56f18606da57dc87dff9f9895a213cef</td>\n",
       "      <td>2018-07-29 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>816</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8b2ab61d02b6e9ae2831fa0d4b3acde7efebd24e</td>\n",
       "      <td>2018-07-25 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>762</td>\n",
       "      <td>650</td>\n",
       "      <td>ID</td>\n",
       "      <td>Rathdrum</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fddfa797b5c1f27cc595cdf5977e580409748fa3</td>\n",
       "      <td>2018-07-30 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>786</td>\n",
       "      <td>840</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>90dcbe67f8b9c2bf761a9ef2f5e6c0a999f4d211</td>\n",
       "      <td>2018-07-09 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>692</td>\n",
       "      <td>642</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145861</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-07-30 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145862</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-07-27 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145863</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-07-27 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145864</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-07-31 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145865</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-07-27 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135546 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       9ece5a07a140346d57db4e2e4479442e46c86fe1  2018-07-20 15:25:00 UTC   \n",
       "1       9d62ba3a56f18606da57dc87dff9f9895a213cef  2018-07-29 18:50:00 UTC   \n",
       "2       8b2ab61d02b6e9ae2831fa0d4b3acde7efebd24e  2018-07-25 13:45:00 UTC   \n",
       "3       fddfa797b5c1f27cc595cdf5977e580409748fa3  2018-07-30 18:50:00 UTC   \n",
       "4       90dcbe67f8b9c2bf761a9ef2f5e6c0a999f4d211  2018-07-09 15:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "145861  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-07-30 15:45:00 UTC   \n",
       "145862  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-07-27 14:05:00 UTC   \n",
       "145863  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-07-27 17:00:00 UTC   \n",
       "145864  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-07-31 18:10:00 UTC   \n",
       "145865  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-07-27 13:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               742                      754   \n",
       "1          cool          hold               816                      840   \n",
       "2          auto          hold               734                      762   \n",
       "3          cool          hold               786                      840   \n",
       "4          auto          hold               694                      692   \n",
       "...         ...           ...               ...                      ...   \n",
       "145861     cool          auto               761                      760   \n",
       "145862     cool          auto               761                      760   \n",
       "145863     cool          auto               767                      760   \n",
       "145864     cool          auto               767                      760   \n",
       "145865     cool          auto               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           665            ID       Chubbuck   \n",
       "1                           790            ID          Boise   \n",
       "2                           650            ID       Rathdrum   \n",
       "3                           700            ID          Boise   \n",
       "4                           642            ID          Boise   \n",
       "...                         ...           ...            ...   \n",
       "145861                      760            ID  Mountain Home   \n",
       "145862                      760            ID  Mountain Home   \n",
       "145863                      760            ID  Mountain Home   \n",
       "145864                      760            ID  Mountain Home   \n",
       "145865                      760            ID  Mountain Home   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        15             False         False            False   \n",
       "1                        30              True         False             True   \n",
       "2                         0             False         False            False   \n",
       "3                        30              True         False             True   \n",
       "4                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "145861                    5             False         False            False   \n",
       "145862                    5             False         False            False   \n",
       "145863                    5             False         False            False   \n",
       "145864                    5             False         False            False   \n",
       "145865                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "145861                       Gas  \n",
       "145862                       Gas  \n",
       "145863                       Gas  \n",
       "145864                       Gas  \n",
       "145865                       Gas  \n",
       "\n",
       "[135546 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/ID/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/ID-day/2019-jul-day-ID.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6569c3ed3ece6db67cac4903e21b2487bbe8a272</td>\n",
       "      <td>2019-07-23 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>788</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6569c3ed3ece6db67cac4903e21b2487bbe8a272</td>\n",
       "      <td>2019-07-21 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ed456d483408d5b0cbe21a04e24a1b05d5a23976</td>\n",
       "      <td>2019-07-03 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>707</td>\n",
       "      <td>657</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d Alene</td>\n",
       "      <td>7</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6569c3ed3ece6db67cac4903e21b2487bbe8a272</td>\n",
       "      <td>2019-07-27 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>730</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>23551e13f2f6837c2c207f4fc4022abf8825f3f7</td>\n",
       "      <td>2019-07-14 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>721</td>\n",
       "      <td>721</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220545</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2019-07-18 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220546</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2019-07-31 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220547</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2019-07-07 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>650</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220548</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2019-07-22 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220549</th>\n",
       "      <td>c6965baccac68c3f496346b377af9ae1a6453b6d</td>\n",
       "      <td>2019-07-24 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Star</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>208705 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6569c3ed3ece6db67cac4903e21b2487bbe8a272  2019-07-23 19:00:00 UTC   \n",
       "1       6569c3ed3ece6db67cac4903e21b2487bbe8a272  2019-07-21 15:35:00 UTC   \n",
       "2       ed456d483408d5b0cbe21a04e24a1b05d5a23976  2019-07-03 17:35:00 UTC   \n",
       "3       6569c3ed3ece6db67cac4903e21b2487bbe8a272  2019-07-27 14:15:00 UTC   \n",
       "4       23551e13f2f6837c2c207f4fc4022abf8825f3f7  2019-07-14 19:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "220545  c6965baccac68c3f496346b377af9ae1a6453b6d  2019-07-18 16:15:00 UTC   \n",
       "220546  c6965baccac68c3f496346b377af9ae1a6453b6d  2019-07-31 16:50:00 UTC   \n",
       "220547  c6965baccac68c3f496346b377af9ae1a6453b6d  2019-07-07 14:00:00 UTC   \n",
       "220548  c6965baccac68c3f496346b377af9ae1a6453b6d  2019-07-22 17:00:00 UTC   \n",
       "220549  c6965baccac68c3f496346b377af9ae1a6453b6d  2019-07-24 15:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               788                      840   \n",
       "1          cool          auto               719                      840   \n",
       "2          auto          hold               710                      707   \n",
       "3          cool          auto               730                      840   \n",
       "4          cool          hold               725                      721   \n",
       "...         ...           ...               ...                      ...   \n",
       "220545     auto          hold               701                      700   \n",
       "220546     auto          hold               702                      700   \n",
       "220547     auto          hold               698                      700   \n",
       "220548     auto          hold               699                      700   \n",
       "220549     auto          hold               699                      700   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           790            ID          Boise   \n",
       "1                           790            ID          Boise   \n",
       "2                           657            ID  Coeur d Alene   \n",
       "3                           790            ID          Boise   \n",
       "4                           721            ID       Meridian   \n",
       "...                         ...           ...            ...   \n",
       "220545                      630            ID           Star   \n",
       "220546                      630            ID           Star   \n",
       "220547                      650            ID           Star   \n",
       "220548                      630            ID           Star   \n",
       "220549                      630            ID           Star   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "2                         7              True         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "220545                    5             False         False            False   \n",
       "220546                    5             False         False            False   \n",
       "220547                    5             False         False            False   \n",
       "220548                    5             False         False            False   \n",
       "220549                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "220545                       Gas  \n",
       "220546                       Gas  \n",
       "220547                       Gas  \n",
       "220548                       Gas  \n",
       "220549                       Gas  \n",
       "\n",
       "[208705 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/ID/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/ID-day/2020-jul-day-ID.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6</td>\n",
       "      <td>2020-07-20 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6</td>\n",
       "      <td>2020-07-20 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6</td>\n",
       "      <td>2020-07-13 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>720</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6</td>\n",
       "      <td>2020-07-13 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>720</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6</td>\n",
       "      <td>2020-07-13 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>720</td>\n",
       "      <td>660</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178641</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2020-07-14 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178642</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2020-07-06 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178643</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2020-07-16 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178644</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2020-07-02 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178645</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2020-07-01 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>174383 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6  2020-07-20 18:35:00 UTC   \n",
       "1       5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6  2020-07-20 16:25:00 UTC   \n",
       "2       5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6  2020-07-13 14:00:00 UTC   \n",
       "3       5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6  2020-07-13 15:00:00 UTC   \n",
       "4       5e194beb4cf255cc5cbf4e821d9f9e05cb7d23d6  2020-07-13 14:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "178641  f592e9a2e74bcee311e16c01f7bf7144a899a803  2020-07-14 15:15:00 UTC   \n",
       "178642  f592e9a2e74bcee311e16c01f7bf7144a899a803  2020-07-06 15:20:00 UTC   \n",
       "178643  f592e9a2e74bcee311e16c01f7bf7144a899a803  2020-07-16 16:35:00 UTC   \n",
       "178644  f592e9a2e74bcee311e16c01f7bf7144a899a803  2020-07-02 16:35:00 UTC   \n",
       "178645  f592e9a2e74bcee311e16c01f7bf7144a899a803  2020-07-01 16:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               720                      720   \n",
       "1          auto          hold               720                      720   \n",
       "2          auto          hold               713                      720   \n",
       "3          auto          hold               724                      720   \n",
       "4          auto          hold               723                      720   \n",
       "...         ...           ...               ...                      ...   \n",
       "178641     cool          auto               703                      760   \n",
       "178642     cool          hold               709                      760   \n",
       "178643     cool          auto               723                      760   \n",
       "178644     cool          hold               694                      760   \n",
       "178645     cool          hold               689                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           670            ID  Garden City   \n",
       "1                           670            ID  Garden City   \n",
       "2                           660            ID  Garden City   \n",
       "3                           660            ID  Garden City   \n",
       "4                           660            ID  Garden City   \n",
       "...                         ...           ...          ...   \n",
       "178641                      670            ID  Garden City   \n",
       "178642                      670            ID  Garden City   \n",
       "178643                      670            ID  Garden City   \n",
       "178644                      670            ID  Garden City   \n",
       "178645                      670            ID  Garden City   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "178641                    0             False         False            False   \n",
       "178642                    0             False         False            False   \n",
       "178643                    0             False         False            False   \n",
       "178644                    0             False         False            False   \n",
       "178645                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "178641                       Gas  \n",
       "178642                       Gas  \n",
       "178643                       Gas  \n",
       "178644                       Gas  \n",
       "178645                       Gas  \n",
       "\n",
       "[174383 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/ID/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/ID-day/2021-jul-day-ID.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e6eb197f10740f7d58d52b8cbb59b54bc7239aa7</td>\n",
       "      <td>2021-07-21 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ea9f239842d6f8438350e9d5e8c375a0fd39a8f0</td>\n",
       "      <td>2021-07-12 09:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>808</td>\n",
       "      <td>830</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33da97b2d600d334d5e19e341aa04519ffe30d89</td>\n",
       "      <td>2021-07-19 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>713</td>\n",
       "      <td>713</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a089a3f30cefc901403c656b872e277e8f39b4b5</td>\n",
       "      <td>2021-07-18 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>726</td>\n",
       "      <td>666</td>\n",
       "      <td>ID</td>\n",
       "      <td>Pocatello</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0a36c8882151c1e485ff07cdbda65bd3bb0a8e17</td>\n",
       "      <td>2021-07-13 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>59</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106569</th>\n",
       "      <td>baaf82bb18030452dfbb504f68ac11d238faa204</td>\n",
       "      <td>2021-07-12 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106570</th>\n",
       "      <td>baaf82bb18030452dfbb504f68ac11d238faa204</td>\n",
       "      <td>2021-07-05 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106571</th>\n",
       "      <td>baaf82bb18030452dfbb504f68ac11d238faa204</td>\n",
       "      <td>2021-07-01 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106572</th>\n",
       "      <td>baaf82bb18030452dfbb504f68ac11d238faa204</td>\n",
       "      <td>2021-07-22 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106573</th>\n",
       "      <td>baaf82bb18030452dfbb504f68ac11d238faa204</td>\n",
       "      <td>2021-07-12 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97877 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e6eb197f10740f7d58d52b8cbb59b54bc7239aa7  2021-07-21 17:55:00 UTC   \n",
       "1       ea9f239842d6f8438350e9d5e8c375a0fd39a8f0  2021-07-12 09:00:00 UTC   \n",
       "2       33da97b2d600d334d5e19e341aa04519ffe30d89  2021-07-19 13:50:00 UTC   \n",
       "5       a089a3f30cefc901403c656b872e277e8f39b4b5  2021-07-18 19:25:00 UTC   \n",
       "6       0a36c8882151c1e485ff07cdbda65bd3bb0a8e17  2021-07-13 15:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "106569  baaf82bb18030452dfbb504f68ac11d238faa204  2021-07-12 16:10:00 UTC   \n",
       "106570  baaf82bb18030452dfbb504f68ac11d238faa204  2021-07-05 19:05:00 UTC   \n",
       "106571  baaf82bb18030452dfbb504f68ac11d238faa204  2021-07-01 14:00:00 UTC   \n",
       "106572  baaf82bb18030452dfbb504f68ac11d238faa204  2021-07-22 17:25:00 UTC   \n",
       "106573  baaf82bb18030452dfbb504f68ac11d238faa204  2021-07-12 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               710                      705   \n",
       "1          auto          hold               808                      830   \n",
       "2          cool          hold               726                      713   \n",
       "5          auto          hold               739                      726   \n",
       "6          cool          hold               723                      705   \n",
       "...         ...           ...               ...                      ...   \n",
       "106569     auto          hold               731                      760   \n",
       "106570     auto          hold               765                      760   \n",
       "106571     auto          hold               752                      760   \n",
       "106572     auto          hold               735                      760   \n",
       "106573     auto          hold               729                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           705            ID   Meridian                   19   \n",
       "1                           720            ID      Boise                   35   \n",
       "2                           713            ID   Meridian                    5   \n",
       "5                           666            ID  Pocatello                   25   \n",
       "6                           705            ID      Boise                   59   \n",
       "...                         ...           ...        ...                  ...   \n",
       "106569                      670            ID      Boise                   10   \n",
       "106570                      670            ID      Boise                   10   \n",
       "106571                      670            ID      Boise                   10   \n",
       "106572                      670            ID      Boise                   10   \n",
       "106573                      670            ID      Boise                   10   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                   True         False            False   \n",
       "1                  False         False             True   \n",
       "2                  False         False            False   \n",
       "5                   True         False            False   \n",
       "6                  False         False             True   \n",
       "...                  ...           ...              ...   \n",
       "106569             False         False            False   \n",
       "106570             False         False            False   \n",
       "106571             False         False            False   \n",
       "106572             False         False            False   \n",
       "106573             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "5                            Gas  \n",
       "6                       Electric  \n",
       "...                          ...  \n",
       "106569                       Gas  \n",
       "106570                       Gas  \n",
       "106571                       Gas  \n",
       "106572                       Gas  \n",
       "106573                       Gas  \n",
       "\n",
       "[97877 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/ID/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00047ac181d6c519a7980acd6dcb0fac57218cf1</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>732.966667</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>720.714286</td>\n",
       "      <td>710.110390</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>731.883333</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>704.796610</td>\n",
       "      <td>700.699153</td>\n",
       "      <td>665.084746</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Firth</td>\n",
       "      <td>722.000000</td>\n",
       "      <td>723.611111</td>\n",
       "      <td>637.069444</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>740.795410</td>\n",
       "      <td>744.630820</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>f8b7152bf32e82564e4ddcc14dc06beee0cb6f2d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>722.287671</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>f9e3f96b2929d85dd24eac09985e5d424fcd02de</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>745.004608</td>\n",
       "      <td>740.322581</td>\n",
       "      <td>740.322581</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>705.680000</td>\n",
       "      <td>701.666667</td>\n",
       "      <td>701.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>732.565528</td>\n",
       "      <td>732.895949</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1238 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00047ac181d6c519a7980acd6dcb0fac57218cf1   jul  2017     cool   \n",
       "1    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   jul  2017     cool   \n",
       "2    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   jul  2017     cool   \n",
       "3    051bb1ef790fd215de91e0304b4f8e06038c2f7d   jul  2017     cool   \n",
       "4    08e17f959d8fff4ab5f1bab6028a513900f97800   jul  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "163  f592e9a2e74bcee311e16c01f7bf7144a899a803   jul  2021     cool   \n",
       "164  f8b7152bf32e82564e4ddcc14dc06beee0cb6f2d   jul  2021     cool   \n",
       "165  f9e3f96b2929d85dd24eac09985e5d424fcd02de   jul  2021     cool   \n",
       "166  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   jul  2021     cool   \n",
       "167  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   jul  2021     auto   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            hold  Idaho Falls            732.966667   \n",
       "1            auto        Eagle            720.714286   \n",
       "2            hold        Eagle            731.883333   \n",
       "3            auto    middleton            704.796610   \n",
       "4            auto        Firth            722.000000   \n",
       "..            ...          ...                   ...   \n",
       "163          hold  Garden City            740.795410   \n",
       "164          hold        Eagle            722.287671   \n",
       "165          hold        Boise            745.004608   \n",
       "166          hold         Kuna            705.680000   \n",
       "167          hold       Burley            732.565528   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     738.000000                   738.000000   \n",
       "1                     710.110390                   690.000000   \n",
       "2                     730.000000                   730.000000   \n",
       "3                     700.699153                   665.084746   \n",
       "4                     723.611111                   637.069444   \n",
       "..                           ...                          ...   \n",
       "163                   744.630820                   670.000000   \n",
       "164                   730.000000                   730.000000   \n",
       "165                   740.322581                   740.322581   \n",
       "166                   701.666667                   701.666667   \n",
       "167                   732.895949                   680.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    7.0             False         False            False  \n",
       "1                   16.0             False         False            False  \n",
       "2                   16.0             False         False            False  \n",
       "3                   10.0              True         False            False  \n",
       "4                   50.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "163                  0.0             False         False            False  \n",
       "164                  5.0             False         False            False  \n",
       "165                  0.0             False         False            False  \n",
       "166                  0.0             False         False            False  \n",
       "167                 60.0              True         False             True  \n",
       "\n",
       "[1238 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/jul/\" + file)\n",
    "    ID_jul = pd.concat([ID_jul, df])\n",
    "    \n",
    "ID_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_jul.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/ID-day/2017-aug-day-ID.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>2017-08-30 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>744</td>\n",
       "      <td>648</td>\n",
       "      <td>ID</td>\n",
       "      <td>Firth</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>deac09f3a312d7d2e2d7895eb83dada1106ee7ba</td>\n",
       "      <td>2017-08-29 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b34646ffdc64e0661b911ef6a6861f06f76556ec</td>\n",
       "      <td>2017-08-20 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>ID</td>\n",
       "      <td>Ammon</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e1f62635f434f12798eec85220f3c6cf220f56e9</td>\n",
       "      <td>2017-08-26 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>742</td>\n",
       "      <td>632</td>\n",
       "      <td>ID</td>\n",
       "      <td>Pocatello</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d3c131ce6f50b46e0c93ac3dbe8f6244e3c51875</td>\n",
       "      <td>2017-08-30 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>799</td>\n",
       "      <td>604</td>\n",
       "      <td>ID</td>\n",
       "      <td>twin falls</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85134</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-08-21 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85135</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-08-21 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85136</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-08-21 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85137</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-08-21 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85138</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2017-08-21 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>83947 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      08e17f959d8fff4ab5f1bab6028a513900f97800  2017-08-30 17:50:00 UTC   \n",
       "1      deac09f3a312d7d2e2d7895eb83dada1106ee7ba  2017-08-29 19:30:00 UTC   \n",
       "2      b34646ffdc64e0661b911ef6a6861f06f76556ec  2017-08-20 13:20:00 UTC   \n",
       "3      e1f62635f434f12798eec85220f3c6cf220f56e9  2017-08-26 19:50:00 UTC   \n",
       "4      d3c131ce6f50b46e0c93ac3dbe8f6244e3c51875  2017-08-30 14:25:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "85134  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-08-21 18:55:00 UTC   \n",
       "85135  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-08-21 14:40:00 UTC   \n",
       "85136  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-08-21 16:10:00 UTC   \n",
       "85137  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-08-21 13:15:00 UTC   \n",
       "85138  f31efa9aa7cd897578fad05d78dedab644e9e923  2017-08-21 14:25:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         cool          auto               686                      744   \n",
       "1         cool          hold               738                      742   \n",
       "2         auto          hold               694                      705   \n",
       "3         auto          auto               739                      742   \n",
       "4         cool          hold               711                      799   \n",
       "...        ...           ...               ...                      ...   \n",
       "85134     cool          hold               742                      760   \n",
       "85135     cool          hold               736                      760   \n",
       "85136     cool          hold               736                      760   \n",
       "85137     cool          hold               739                      760   \n",
       "85138     cool          hold               737                      760   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                          648            ID          Firth   \n",
       "1                          742            ID          Boise   \n",
       "2                          655            ID          Ammon   \n",
       "3                          632            ID      Pocatello   \n",
       "4                          604            ID     twin falls   \n",
       "...                        ...           ...            ...   \n",
       "85134                      760            ID  Mountain Home   \n",
       "85135                      760            ID  Mountain Home   \n",
       "85136                      760            ID  Mountain Home   \n",
       "85137                      760            ID  Mountain Home   \n",
       "85138                      760            ID  Mountain Home   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       50             False         False            False   \n",
       "1                        5             False         False            False   \n",
       "2                        5             False         False            False   \n",
       "3                       40             False         False             True   \n",
       "4                       10              True         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "85134                    5             False         False            False   \n",
       "85135                    5             False         False            False   \n",
       "85136                    5             False         False            False   \n",
       "85137                    5             False         False            False   \n",
       "85138                    5             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                      Electric  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "85134                       Gas  \n",
       "85135                       Gas  \n",
       "85136                       Gas  \n",
       "85137                       Gas  \n",
       "85138                       Gas  \n",
       "\n",
       "[83947 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/ID/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/ID-day/2018-aug-day-ID.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60698929712d7907c3ae822cf56f4d2f3d211504</td>\n",
       "      <td>2018-08-18 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>60698929712d7907c3ae822cf56f4d2f3d211504</td>\n",
       "      <td>2018-08-18 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>662</td>\n",
       "      <td>670</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60698929712d7907c3ae822cf56f4d2f3d211504</td>\n",
       "      <td>2018-08-18 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>666</td>\n",
       "      <td>670</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>60698929712d7907c3ae822cf56f4d2f3d211504</td>\n",
       "      <td>2018-08-18 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>671</td>\n",
       "      <td>670</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>63e7d6b818b36d58ffe102f1b216b11c73df0734</td>\n",
       "      <td>2018-08-07 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Caldwell</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162961</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-08-02 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162962</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-08-17 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162963</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-08-10 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162964</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-08-24 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162965</th>\n",
       "      <td>f31efa9aa7cd897578fad05d78dedab644e9e923</td>\n",
       "      <td>2018-08-08 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Mountain Home</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>151452 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       60698929712d7907c3ae822cf56f4d2f3d211504  2018-08-18 15:30:00 UTC   \n",
       "1       60698929712d7907c3ae822cf56f4d2f3d211504  2018-08-18 14:30:00 UTC   \n",
       "2       60698929712d7907c3ae822cf56f4d2f3d211504  2018-08-18 15:50:00 UTC   \n",
       "3       60698929712d7907c3ae822cf56f4d2f3d211504  2018-08-18 14:55:00 UTC   \n",
       "4       63e7d6b818b36d58ffe102f1b216b11c73df0734  2018-08-07 16:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "162961  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-08-02 16:50:00 UTC   \n",
       "162962  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-08-17 14:45:00 UTC   \n",
       "162963  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-08-10 13:25:00 UTC   \n",
       "162964  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-08-24 16:40:00 UTC   \n",
       "162965  f31efa9aa7cd897578fad05d78dedab644e9e923  2018-08-08 19:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               670                      670   \n",
       "1          auto          auto               662                      670   \n",
       "2          auto          auto               666                      670   \n",
       "3          auto          auto               671                      670   \n",
       "4          cool          auto               669                      670   \n",
       "...         ...           ...               ...                      ...   \n",
       "162961     cool          auto               759                      760   \n",
       "162962     cool          hold               763                      760   \n",
       "162963     cool          hold               760                      760   \n",
       "162964     cool          hold               753                      760   \n",
       "162965     cool          hold               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           620            ID       Meridian   \n",
       "1                           620            ID       Meridian   \n",
       "2                           620            ID       Meridian   \n",
       "3                           620            ID       Meridian   \n",
       "4                           670            ID       Caldwell   \n",
       "...                         ...           ...            ...   \n",
       "162961                      760            ID  Mountain Home   \n",
       "162962                      760            ID  Mountain Home   \n",
       "162963                      760            ID  Mountain Home   \n",
       "162964                      760            ID  Mountain Home   \n",
       "162965                      760            ID  Mountain Home   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        28             False         False            False   \n",
       "1                        28             False         False            False   \n",
       "2                        28             False         False            False   \n",
       "3                        28             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "162961                    5             False         False            False   \n",
       "162962                    5             False         False            False   \n",
       "162963                    5             False         False            False   \n",
       "162964                    5             False         False            False   \n",
       "162965                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "162961                       Gas  \n",
       "162962                       Gas  \n",
       "162963                       Gas  \n",
       "162964                       Gas  \n",
       "162965                       Gas  \n",
       "\n",
       "[151452 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/ID/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/ID-day/2019-aug-day-ID.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>2019-08-03 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ed456d483408d5b0cbe21a04e24a1b05d5a23976</td>\n",
       "      <td>2019-08-31 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>717</td>\n",
       "      <td>667</td>\n",
       "      <td>ID</td>\n",
       "      <td>Coeur d Alene</td>\n",
       "      <td>7</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9d62ba3a56f18606da57dc87dff9f9895a213cef</td>\n",
       "      <td>2019-08-20 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>795</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9d62ba3a56f18606da57dc87dff9f9895a213cef</td>\n",
       "      <td>2019-08-21 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>810</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1d13858b9384d00026f0138515607756f8ddad50</td>\n",
       "      <td>2019-08-14 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211494</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-08-12 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211495</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-08-13 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211496</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-08-25 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211497</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-08-14 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211498</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-08-11 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>199916 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fa460aad6a17d2a235c05c076ddfc6e429ff15b8  2019-08-03 13:05:00 UTC   \n",
       "1       ed456d483408d5b0cbe21a04e24a1b05d5a23976  2019-08-31 17:25:00 UTC   \n",
       "3       9d62ba3a56f18606da57dc87dff9f9895a213cef  2019-08-20 13:50:00 UTC   \n",
       "4       9d62ba3a56f18606da57dc87dff9f9895a213cef  2019-08-21 16:25:00 UTC   \n",
       "8       1d13858b9384d00026f0138515607756f8ddad50  2019-08-14 12:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "211494  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-08-12 13:20:00 UTC   \n",
       "211495  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-08-13 12:40:00 UTC   \n",
       "211496  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-08-25 12:40:00 UTC   \n",
       "211497  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-08-14 12:50:00 UTC   \n",
       "211498  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-08-11 13:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               698                      718   \n",
       "1          auto          hold               714                      717   \n",
       "3          cool          hold               795                      840   \n",
       "4          cool          hold               810                      840   \n",
       "8          cool          hold               704                      709   \n",
       "...         ...           ...               ...                      ...   \n",
       "211494     auto          auto               710                      760   \n",
       "211495     auto          auto               705                      760   \n",
       "211496     auto          auto               725                      760   \n",
       "211497     auto          hold               720                      760   \n",
       "211498     auto          hold               711                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           718            ID          Boise   \n",
       "1                           667            ID  Coeur d Alene   \n",
       "3                           790            ID          Boise   \n",
       "4                           790            ID          Boise   \n",
       "8                           709            ID          Boise   \n",
       "...                         ...           ...            ...   \n",
       "211494                      670            ID    Garden City   \n",
       "211495                      670            ID    Garden City   \n",
       "211496                      670            ID    Garden City   \n",
       "211497                      670            ID    Garden City   \n",
       "211498                      670            ID    Garden City   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                         7              True         False            False   \n",
       "3                        30              True         False             True   \n",
       "4                        30              True         False             True   \n",
       "8                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "211494                    0             False         False            False   \n",
       "211495                    0             False         False            False   \n",
       "211496                    0             False         False            False   \n",
       "211497                    0             False         False            False   \n",
       "211498                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "8                            Gas  \n",
       "...                          ...  \n",
       "211494                       Gas  \n",
       "211495                       Gas  \n",
       "211496                       Gas  \n",
       "211497                       Gas  \n",
       "211498                       Gas  \n",
       "\n",
       "[199916 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/ID/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/ID-day/2020-aug-day-ID.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>07169235e0908f68733f94d727dd8b31b98f1f2b</td>\n",
       "      <td>2020-08-16 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Pocatello</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>07169235e0908f68733f94d727dd8b31b98f1f2b</td>\n",
       "      <td>2020-08-14 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Pocatello</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b7a3c538b12ccba34119d3970e8f4b64f1c37cdc</td>\n",
       "      <td>2020-08-27 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Caldwell</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b7a3c538b12ccba34119d3970e8f4b64f1c37cdc</td>\n",
       "      <td>2020-08-27 16:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Caldwell</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>07169235e0908f68733f94d727dd8b31b98f1f2b</td>\n",
       "      <td>2020-08-14 15:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Pocatello</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166258</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-08-09 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166259</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-08-08 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166260</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-08-09 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166261</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-08-09 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166262</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>2020-08-09 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Burley</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>158647 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       07169235e0908f68733f94d727dd8b31b98f1f2b  2020-08-16 13:55:00 UTC   \n",
       "1       07169235e0908f68733f94d727dd8b31b98f1f2b  2020-08-14 17:40:00 UTC   \n",
       "2       b7a3c538b12ccba34119d3970e8f4b64f1c37cdc  2020-08-27 15:05:00 UTC   \n",
       "3       b7a3c538b12ccba34119d3970e8f4b64f1c37cdc  2020-08-27 16:00:00 UTC   \n",
       "4       07169235e0908f68733f94d727dd8b31b98f1f2b  2020-08-14 15:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "166258  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-08-09 18:55:00 UTC   \n",
       "166259  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-08-08 15:40:00 UTC   \n",
       "166260  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-08-09 17:45:00 UTC   \n",
       "166261  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-08-09 16:55:00 UTC   \n",
       "166262  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b  2020-08-09 18:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               672                      670   \n",
       "1          cool          auto               672                      670   \n",
       "2          cool          hold               669                      670   \n",
       "3          cool          hold               672                      670   \n",
       "4          cool          auto               675                      670   \n",
       "...         ...           ...               ...                      ...   \n",
       "166258     auto          hold               734                      760   \n",
       "166259     auto          hold               727                      760   \n",
       "166260     auto          hold               731                      760   \n",
       "166261     auto          hold               731                      760   \n",
       "166262     auto          hold               731                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           670            ID  Pocatello                   65   \n",
       "1                           670            ID  Pocatello                   65   \n",
       "2                           670            ID   Caldwell                   29   \n",
       "3                           670            ID   Caldwell                   29   \n",
       "4                           670            ID  Pocatello                   65   \n",
       "...                         ...           ...        ...                  ...   \n",
       "166258                      680            ID     Burley                   60   \n",
       "166259                      680            ID     Burley                   60   \n",
       "166260                      680            ID     Burley                   60   \n",
       "166261                      680            ID     Burley                   60   \n",
       "166262                      680            ID     Burley                   60   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "166258              True         False             True   \n",
       "166259              True         False             True   \n",
       "166260              True         False             True   \n",
       "166261              True         False             True   \n",
       "166262              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "166258                  Electric  \n",
       "166259                  Electric  \n",
       "166260                  Electric  \n",
       "166261                  Electric  \n",
       "166262                  Electric  \n",
       "\n",
       "[158647 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/ID/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00047ac181d6c519a7980acd6dcb0fac57218cf1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>721.545455</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>729.222222</td>\n",
       "      <td>720.888889</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>703.343313</td>\n",
       "      <td>702.359281</td>\n",
       "      <td>666.966068</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Firth</td>\n",
       "      <td>678.192308</td>\n",
       "      <td>680.871795</td>\n",
       "      <td>642.602564</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Firth</td>\n",
       "      <td>684.787037</td>\n",
       "      <td>690.632716</td>\n",
       "      <td>689.580247</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>fbfe005e468e4b04640a3a1efda8c6624aec4a76</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>751.392857</td>\n",
       "      <td>736.535714</td>\n",
       "      <td>665.250000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>fbfe005e468e4b04640a3a1efda8c6624aec4a76</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>774.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>297</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Burley</td>\n",
       "      <td>737.972041</td>\n",
       "      <td>740.004660</td>\n",
       "      <td>680.013979</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>737.316713</td>\n",
       "      <td>740.320927</td>\n",
       "      <td>681.748596</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>299</th>\n",
       "      <td>ff576006ddec2339bd9c9bcbef140daeba69bae1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>732.707761</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>739.990717</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>977 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00047ac181d6c519a7980acd6dcb0fac57218cf1   aug  2017     cool   \n",
       "1    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   aug  2017     cool   \n",
       "2    051bb1ef790fd215de91e0304b4f8e06038c2f7d   aug  2017     cool   \n",
       "3    08e17f959d8fff4ab5f1bab6028a513900f97800   aug  2017     cool   \n",
       "4    08e17f959d8fff4ab5f1bab6028a513900f97800   aug  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "295  fbfe005e468e4b04640a3a1efda8c6624aec4a76   aug  2020     auto   \n",
       "296  fbfe005e468e4b04640a3a1efda8c6624aec4a76   aug  2020     cool   \n",
       "297  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   aug  2020     auto   \n",
       "298  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   aug  2020     auto   \n",
       "299  ff576006ddec2339bd9c9bcbef140daeba69bae1   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto  Idaho Falls            721.545455   \n",
       "1            auto        Eagle            729.222222   \n",
       "2            auto    middleton            703.343313   \n",
       "3            auto        Firth            678.192308   \n",
       "4            hold        Firth            684.787037   \n",
       "..            ...          ...                   ...   \n",
       "295          hold     Meridian            751.392857   \n",
       "296          hold     Meridian            774.000000   \n",
       "297          auto       Burley            737.972041   \n",
       "298          hold       Burley            737.316713   \n",
       "299          auto     Meridian            732.707761   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     740.000000                   740.000000   \n",
       "1                     720.888889                   690.000000   \n",
       "2                     702.359281                   666.966068   \n",
       "3                     680.871795                   642.602564   \n",
       "4                     690.632716                   689.580247   \n",
       "..                           ...                          ...   \n",
       "295                   736.535714                   665.250000   \n",
       "296                   700.000000                   700.000000   \n",
       "297                   740.004660                   680.013979   \n",
       "298                   740.320927                   681.748596   \n",
       "299                   740.000000                   739.990717   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    7.0             False         False            False  \n",
       "1                   16.0             False         False            False  \n",
       "2                   10.0              True         False            False  \n",
       "3                   50.0             False         False            False  \n",
       "4                   50.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "295                  0.0             False         False            False  \n",
       "296                  0.0             False         False            False  \n",
       "297                 60.0              True         False             True  \n",
       "298                 60.0              True         False             True  \n",
       "299                 17.0             False         False            False  \n",
       "\n",
       "[977 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/aug/\" + file)\n",
    "    ID_aug = pd.concat([ID_aug, df])\n",
    "    \n",
    "ID_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_aug.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/ID-day/2017-dec-day-ID.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2017-12-19 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2017-12-27 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2017-12-26 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>af2e12e051baa0782690f8e4fb33211b8c94ad0a</td>\n",
       "      <td>2017-12-29 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>beff863fed65578068af09c0850c160b1bbac7fe</td>\n",
       "      <td>2017-12-10 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>754</td>\n",
       "      <td>714</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104074</th>\n",
       "      <td>a123897f6679cea5de1b3e15dbed0426117ef1b0</td>\n",
       "      <td>2017-12-03 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Rexburg</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104075</th>\n",
       "      <td>a123897f6679cea5de1b3e15dbed0426117ef1b0</td>\n",
       "      <td>2017-12-03 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Rexburg</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104076</th>\n",
       "      <td>a123897f6679cea5de1b3e15dbed0426117ef1b0</td>\n",
       "      <td>2017-12-02 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Rexburg</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104077</th>\n",
       "      <td>a4c296c973cad2f47143947c472b96fd8c2a1c5f</td>\n",
       "      <td>2017-12-06 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>692</td>\n",
       "      <td>ID</td>\n",
       "      <td>eagle</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104078</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2017-12-25 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100834 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       af2e12e051baa0782690f8e4fb33211b8c94ad0a  2017-12-19 15:45:00 UTC   \n",
       "1       af2e12e051baa0782690f8e4fb33211b8c94ad0a  2017-12-27 14:30:00 UTC   \n",
       "2       af2e12e051baa0782690f8e4fb33211b8c94ad0a  2017-12-26 16:10:00 UTC   \n",
       "3       af2e12e051baa0782690f8e4fb33211b8c94ad0a  2017-12-29 17:35:00 UTC   \n",
       "4       beff863fed65578068af09c0850c160b1bbac7fe  2017-12-10 16:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "104074  a123897f6679cea5de1b3e15dbed0426117ef1b0  2017-12-03 16:15:00 UTC   \n",
       "104075  a123897f6679cea5de1b3e15dbed0426117ef1b0  2017-12-03 15:40:00 UTC   \n",
       "104076  a123897f6679cea5de1b3e15dbed0426117ef1b0  2017-12-02 19:30:00 UTC   \n",
       "104077  a4c296c973cad2f47143947c472b96fd8c2a1c5f  2017-12-06 17:50:00 UTC   \n",
       "104078  c3116d6bff13952b6ae40d463c2fd6a368b78734  2017-12-25 14:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               684                      840   \n",
       "1          heat          auto               687                      840   \n",
       "2          heat          auto               689                      840   \n",
       "3          heat          auto               694                      840   \n",
       "4          auto          auto               698                      754   \n",
       "...         ...           ...               ...                      ...   \n",
       "104074     auto          hold               709                      760   \n",
       "104075     auto          hold               708                      760   \n",
       "104076     auto          hold               714                      760   \n",
       "104077     heat          hold               694                      760   \n",
       "104078     heat          hold               752                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           690            ID     eagle                    0   \n",
       "1                           690            ID     eagle                    0   \n",
       "2                           690            ID     eagle                    0   \n",
       "3                           690            ID     eagle                    0   \n",
       "4                           714            ID  Meridian                    5   \n",
       "...                         ...           ...       ...                  ...   \n",
       "104074                      710            ID   Rexburg                   15   \n",
       "104075                      710            ID   Rexburg                   15   \n",
       "104076                      710            ID   Rexburg                   15   \n",
       "104077                      692            ID     eagle                   10   \n",
       "104078                      760            ID     Boise                   20   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                   True          True             True   \n",
       "1                   True          True             True   \n",
       "2                   True          True             True   \n",
       "3                   True          True             True   \n",
       "4                   True         False            False   \n",
       "...                  ...           ...              ...   \n",
       "104074             False         False            False   \n",
       "104075             False         False            False   \n",
       "104076             False         False            False   \n",
       "104077             False         False            False   \n",
       "104078             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "104074                       Gas  \n",
       "104075                       Gas  \n",
       "104076                       Gas  \n",
       "104077                       Gas  \n",
       "104078                       Gas  \n",
       "\n",
       "[100834 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/ID/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/ID-day/2018-dec-day-ID.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21</td>\n",
       "      <td>2018-12-15 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>776</td>\n",
       "      <td>726</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21</td>\n",
       "      <td>2018-12-23 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>ID</td>\n",
       "      <td>Twin Falls</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9ece5a07a140346d57db4e2e4479442e46c86fe1</td>\n",
       "      <td>2018-12-23 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>784</td>\n",
       "      <td>685</td>\n",
       "      <td>ID</td>\n",
       "      <td>Chubbuck</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d9128a801171af6d2f7321cfec7822d7c8399d0f</td>\n",
       "      <td>2018-12-13 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>762</td>\n",
       "      <td>712</td>\n",
       "      <td>ID</td>\n",
       "      <td>Ketchum</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>12ca3d29042b0af6a26775df78b8ed4e5da47e8a</td>\n",
       "      <td>2018-12-10 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151215</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2018-12-22 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151216</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2018-12-30 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151217</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2018-12-26 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151218</th>\n",
       "      <td>82358b1411142867e615ad6713effeeb472ab863</td>\n",
       "      <td>2018-12-22 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151220</th>\n",
       "      <td>f461b8107c90a9bf0149592b711a916dbf040b73</td>\n",
       "      <td>2018-12-10 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>760</td>\n",
       "      <td>677</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>147246 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21  2018-12-15 13:15:00 UTC   \n",
       "1       cd8f3cdef7e2de54650c667bedcd79dd3a9d4b21  2018-12-23 15:00:00 UTC   \n",
       "2       9ece5a07a140346d57db4e2e4479442e46c86fe1  2018-12-23 16:00:00 UTC   \n",
       "3       d9128a801171af6d2f7321cfec7822d7c8399d0f  2018-12-13 19:10:00 UTC   \n",
       "4       12ca3d29042b0af6a26775df78b8ed4e5da47e8a  2018-12-10 17:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "151215  82358b1411142867e615ad6713effeeb472ab863  2018-12-22 15:15:00 UTC   \n",
       "151216  82358b1411142867e615ad6713effeeb472ab863  2018-12-30 16:10:00 UTC   \n",
       "151217  82358b1411142867e615ad6713effeeb472ab863  2018-12-26 16:55:00 UTC   \n",
       "151218  82358b1411142867e615ad6713effeeb472ab863  2018-12-22 16:40:00 UTC   \n",
       "151220  f461b8107c90a9bf0149592b711a916dbf040b73  2018-12-10 15:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               727                      776   \n",
       "1          auto          hold               712                      766   \n",
       "2          auto          hold               676                      784   \n",
       "3          heat          auto               709                      762   \n",
       "4          heat          hold               680                      687   \n",
       "...         ...           ...               ...                      ...   \n",
       "151215     auto          hold               676                      760   \n",
       "151216     auto          hold               670                      760   \n",
       "151217     auto          hold               685                      760   \n",
       "151218     auto          hold               683                      760   \n",
       "151220     auto          hold               672                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           726            ID  Twin Falls   \n",
       "1                           716            ID  Twin Falls   \n",
       "2                           685            ID    Chubbuck   \n",
       "3                           712            ID     Ketchum   \n",
       "4                           687            ID    Meridian   \n",
       "...                         ...           ...         ...   \n",
       "151215                      680            ID       Boise   \n",
       "151216                      670            ID       Boise   \n",
       "151217                      690            ID       Boise   \n",
       "151218                      690            ID       Boise   \n",
       "151220                      677            ID       Boise   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "2                        15             False         False            False   \n",
       "3                         5              True         False            False   \n",
       "4                         0              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "151215                   30             False         False            False   \n",
       "151216                   30             False         False            False   \n",
       "151217                   30             False         False            False   \n",
       "151218                   30             False         False            False   \n",
       "151220                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "151215                       Gas  \n",
       "151216                       Gas  \n",
       "151217                       Gas  \n",
       "151218                       Gas  \n",
       "151220                       Gas  \n",
       "\n",
       "[147246 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/ID/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/ID-day/2019-dec-day-ID.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3818b68d3bb9746e331ea025aa36cf68cd83908c</td>\n",
       "      <td>2019-12-22 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>727</td>\n",
       "      <td>720</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f5014ad9bf7b89a5123286b039ab8809df9c859</td>\n",
       "      <td>2019-12-12 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>795</td>\n",
       "      <td>695</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dd69a899c3a34350944a99db82c33ea255a06552</td>\n",
       "      <td>2019-12-10 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>769</td>\n",
       "      <td>679</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93e1fba26e25ad7af5711c4d96856adfbd3dde43</td>\n",
       "      <td>2019-12-18 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dd69a899c3a34350944a99db82c33ea255a06552</td>\n",
       "      <td>2019-12-06 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>769</td>\n",
       "      <td>679</td>\n",
       "      <td>ID</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172596</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-12-07 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>613</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172597</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-12-05 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>625</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172598</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-12-17 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>606</td>\n",
       "      <td>760</td>\n",
       "      <td>610</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172599</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-12-17 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>604</td>\n",
       "      <td>760</td>\n",
       "      <td>610</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172600</th>\n",
       "      <td>f592e9a2e74bcee311e16c01f7bf7144a899a803</td>\n",
       "      <td>2019-12-09 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>616</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>ID</td>\n",
       "      <td>Garden City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>163777 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       3818b68d3bb9746e331ea025aa36cf68cd83908c  2019-12-22 18:10:00 UTC   \n",
       "1       2f5014ad9bf7b89a5123286b039ab8809df9c859  2019-12-12 13:20:00 UTC   \n",
       "2       dd69a899c3a34350944a99db82c33ea255a06552  2019-12-10 14:25:00 UTC   \n",
       "3       93e1fba26e25ad7af5711c4d96856adfbd3dde43  2019-12-18 18:35:00 UTC   \n",
       "4       dd69a899c3a34350944a99db82c33ea255a06552  2019-12-06 17:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "172596  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-12-07 16:05:00 UTC   \n",
       "172597  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-12-05 19:25:00 UTC   \n",
       "172598  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-12-17 16:50:00 UTC   \n",
       "172599  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-12-17 19:20:00 UTC   \n",
       "172600  f592e9a2e74bcee311e16c01f7bf7144a899a803  2019-12-09 15:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               716                      727   \n",
       "1          auto          hold               690                      795   \n",
       "2          auto          hold               675                      769   \n",
       "3          heat          hold               702                      705   \n",
       "4          auto          hold               675                      769   \n",
       "...         ...           ...               ...                      ...   \n",
       "172596     auto          auto               613                      760   \n",
       "172597     auto          hold               625                      760   \n",
       "172598     auto          auto               606                      760   \n",
       "172599     auto          auto               604                      760   \n",
       "172600     auto          hold               616                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           720            ID        Nampa   \n",
       "1                           695            ID  Idaho Falls   \n",
       "2                           679            ID  Idaho Falls   \n",
       "3                           705            ID        Eagle   \n",
       "4                           679            ID  Idaho Falls   \n",
       "...                         ...           ...          ...   \n",
       "172596                      620            ID  Garden City   \n",
       "172597                      630            ID  Garden City   \n",
       "172598                      610            ID  Garden City   \n",
       "172599                      610            ID  Garden City   \n",
       "172600                      620            ID  Garden City   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        15             False         False            False   \n",
       "1                        50             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        20              True         False            False   \n",
       "4                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "172596                    0             False         False            False   \n",
       "172597                    0             False         False            False   \n",
       "172598                    0             False         False            False   \n",
       "172599                    0             False         False            False   \n",
       "172600                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "172596                       Gas  \n",
       "172597                       Gas  \n",
       "172598                       Gas  \n",
       "172599                       Gas  \n",
       "172600                       Gas  \n",
       "\n",
       "[163777 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/ID/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/ID-day/2020-dec-day-ID.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>253543009c667cbcfd69450d5405908cb4901372</td>\n",
       "      <td>2020-12-21 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>746</td>\n",
       "      <td>686</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f2fbf8989c48ef7f73662ee36e35ff771cb37bc2</td>\n",
       "      <td>2020-12-17 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>ID</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>46b1845175d7b26c6296f339ba11a9557b248613</td>\n",
       "      <td>2020-12-07 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>ID</td>\n",
       "      <td>Moscow</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>253543009c667cbcfd69450d5405908cb4901372</td>\n",
       "      <td>2020-12-17 12:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>746</td>\n",
       "      <td>686</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3088a5a1b9dd1a19fede9b335c8bd98e56c0ea83</td>\n",
       "      <td>2020-12-12 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>ID</td>\n",
       "      <td>Nampa</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151466</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2020-12-05 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151467</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2020-12-03 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151468</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2020-12-03 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151469</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2020-12-04 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151470</th>\n",
       "      <td>c3116d6bff13952b6ae40d463c2fd6a368b78734</td>\n",
       "      <td>2020-12-04 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>ID</td>\n",
       "      <td>Boise</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>147158 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       253543009c667cbcfd69450d5405908cb4901372  2020-12-21 13:15:00 UTC   \n",
       "1       f2fbf8989c48ef7f73662ee36e35ff771cb37bc2  2020-12-17 17:30:00 UTC   \n",
       "2       46b1845175d7b26c6296f339ba11a9557b248613  2020-12-07 19:40:00 UTC   \n",
       "3       253543009c667cbcfd69450d5405908cb4901372  2020-12-17 12:10:00 UTC   \n",
       "4       3088a5a1b9dd1a19fede9b335c8bd98e56c0ea83  2020-12-12 18:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "151466  c3116d6bff13952b6ae40d463c2fd6a368b78734  2020-12-05 18:45:00 UTC   \n",
       "151467  c3116d6bff13952b6ae40d463c2fd6a368b78734  2020-12-03 14:15:00 UTC   \n",
       "151468  c3116d6bff13952b6ae40d463c2fd6a368b78734  2020-12-03 14:25:00 UTC   \n",
       "151469  c3116d6bff13952b6ae40d463c2fd6a368b78734  2020-12-04 18:15:00 UTC   \n",
       "151470  c3116d6bff13952b6ae40d463c2fd6a368b78734  2020-12-04 14:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               687                      746   \n",
       "1          heat          hold               703                      705   \n",
       "2          heat          hold               677                      675   \n",
       "3          auto          hold               684                      746   \n",
       "4          heat          hold               672                      677   \n",
       "...         ...           ...               ...                      ...   \n",
       "151466     auto          hold               708                      760   \n",
       "151467     auto          hold               691                      760   \n",
       "151468     auto          hold               701                      760   \n",
       "151469     auto          hold               702                      760   \n",
       "151470     auto          hold               699                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           686            ID     Boise                    0   \n",
       "1                           705            ID  Meridian                    5   \n",
       "2                           675            ID    Moscow                   50   \n",
       "3                           686            ID     Boise                    0   \n",
       "4                           677            ID     Nampa                   25   \n",
       "...                         ...           ...       ...                  ...   \n",
       "151466                      710            ID     Boise                   20   \n",
       "151467                      700            ID     Boise                   20   \n",
       "151468                      700            ID     Boise                   20   \n",
       "151469                      710            ID     Boise                   20   \n",
       "151470                      700            ID     Boise                   20   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                   True         False            False   \n",
       "...                  ...           ...              ...   \n",
       "151466             False         False            False   \n",
       "151467             False         False            False   \n",
       "151468             False         False            False   \n",
       "151469             False         False            False   \n",
       "151470             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "151466                       Gas  \n",
       "151467                       Gas  \n",
       "151468                       Gas  \n",
       "151469                       Gas  \n",
       "151470                       Gas  \n",
       "\n",
       "[147158 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/ID/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/ID/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00047ac181d6c519a7980acd6dcb0fac57218cf1</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>707.807692</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>697.391026</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>696.993590</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>697.192308</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0242366947dd8f91e501280859cb1352ac56a3b1</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Shelley</td>\n",
       "      <td>683.679487</td>\n",
       "      <td>685.000000</td>\n",
       "      <td>685.000000</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>694.931973</td>\n",
       "      <td>699.260544</td>\n",
       "      <td>699.260544</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>680.218182</td>\n",
       "      <td>682.272727</td>\n",
       "      <td>682.272727</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>679.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Burley</td>\n",
       "      <td>710.529915</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>310</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>718.139439</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1166 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00047ac181d6c519a7980acd6dcb0fac57218cf1   dec  2017     heat   \n",
       "1    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   dec  2017     auto   \n",
       "2    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   dec  2017     heat   \n",
       "3    01e233c4fce6aabdd3f8be666d5f32f972d2a16b   dec  2017     heat   \n",
       "4    0242366947dd8f91e501280859cb1352ac56a3b1   dec  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "306  fa460aad6a17d2a235c05c076ddfc6e429ff15b8   dec  2020     heat   \n",
       "307  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   dec  2020     heat   \n",
       "308  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   dec  2020     heat   \n",
       "309  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   dec  2020     auto   \n",
       "310  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   dec  2020     auto   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto  Idaho Falls            707.807692   \n",
       "1            hold        Eagle            697.391026   \n",
       "2            auto        Eagle            696.993590   \n",
       "3            hold        Eagle            697.192308   \n",
       "4            auto      Shelley            683.679487   \n",
       "..            ...          ...                   ...   \n",
       "306          hold        Boise            694.931973   \n",
       "307          auto         Kuna            680.218182   \n",
       "308          hold         Kuna            679.000000   \n",
       "309          auto       Burley            710.529915   \n",
       "310          hold       Burley            718.139439   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     710.000000                   710.000000   \n",
       "1                     750.000000                   700.000000   \n",
       "2                     700.000000                   700.000000   \n",
       "3                     700.000000                   700.000000   \n",
       "4                     685.000000                   685.000000   \n",
       "..                           ...                          ...   \n",
       "306                   699.260544                   699.260544   \n",
       "307                   682.272727                   682.272727   \n",
       "308                   680.000000                   680.000000   \n",
       "309                   780.000000                   720.000000   \n",
       "310                   780.000000                   730.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    7.0             False         False            False  \n",
       "1                   16.0             False         False            False  \n",
       "2                   16.0             False         False            False  \n",
       "3                   16.0             False         False            False  \n",
       "4                   40.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "306                  5.0             False         False            False  \n",
       "307                  0.0             False         False            False  \n",
       "308                  0.0             False         False            False  \n",
       "309                 60.0              True         False             True  \n",
       "310                 60.0              True         False             True  \n",
       "\n",
       "[1166 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/ID/dec/\" + file)\n",
    "    ID_dec = pd.concat([ID_dec, df])\n",
    "    \n",
    "ID_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_dec.to_csv(\"Scraper_Output/State_Month_Day/ID/ID_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/ID/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00047ac181d6c519a7980acd6dcb0fac57218cf1</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Idaho Falls</td>\n",
       "      <td>721.545455</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>7.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01e233c4fce6aabdd3f8be666d5f32f972d2a16b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eagle</td>\n",
       "      <td>729.222222</td>\n",
       "      <td>720.888889</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>16.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>051bb1ef790fd215de91e0304b4f8e06038c2f7d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>middleton</td>\n",
       "      <td>703.343313</td>\n",
       "      <td>702.359281</td>\n",
       "      <td>666.966068</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Firth</td>\n",
       "      <td>678.192308</td>\n",
       "      <td>680.871795</td>\n",
       "      <td>642.602564</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>08e17f959d8fff4ab5f1bab6028a513900f97800</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Firth</td>\n",
       "      <td>684.787037</td>\n",
       "      <td>690.632716</td>\n",
       "      <td>689.580247</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1343</th>\n",
       "      <td>f9e3f96b2929d85dd24eac09985e5d424fcd02de</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>735.618312</td>\n",
       "      <td>738.469939</td>\n",
       "      <td>738.399890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1344</th>\n",
       "      <td>fa460aad6a17d2a235c05c076ddfc6e429ff15b8</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Boise</td>\n",
       "      <td>709.340206</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1345</th>\n",
       "      <td>fa9e861361c9be63c34bdb1d26f4718da1e69d4f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kuna</td>\n",
       "      <td>697.672269</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1346</th>\n",
       "      <td>feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Burley</td>\n",
       "      <td>734.242340</td>\n",
       "      <td>746.652925</td>\n",
       "      <td>699.992201</td>\n",
       "      <td>60.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1347</th>\n",
       "      <td>ff576006ddec2339bd9c9bcbef140daeba69bae1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Meridian</td>\n",
       "      <td>735.821429</td>\n",
       "      <td>740.571429</td>\n",
       "      <td>739.571429</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6800 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00047ac181d6c519a7980acd6dcb0fac57218cf1   aug  2017     cool   \n",
       "1     01e233c4fce6aabdd3f8be666d5f32f972d2a16b   aug  2017     cool   \n",
       "2     051bb1ef790fd215de91e0304b4f8e06038c2f7d   aug  2017     cool   \n",
       "3     08e17f959d8fff4ab5f1bab6028a513900f97800   aug  2017     cool   \n",
       "4     08e17f959d8fff4ab5f1bab6028a513900f97800   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1343  f9e3f96b2929d85dd24eac09985e5d424fcd02de   jun  2021     cool   \n",
       "1344  fa460aad6a17d2a235c05c076ddfc6e429ff15b8   jun  2021     auto   \n",
       "1345  fa9e861361c9be63c34bdb1d26f4718da1e69d4f   jun  2021     cool   \n",
       "1346  feafd9d8d0bec05672dcb64f7ee6d7ccf682e75b   jun  2021     auto   \n",
       "1347  ff576006ddec2339bd9c9bcbef140daeba69bae1   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0             auto  Idaho Falls            721.545455   \n",
       "1             auto        Eagle            729.222222   \n",
       "2             auto    middleton            703.343313   \n",
       "3             auto        Firth            678.192308   \n",
       "4             hold        Firth            684.787037   \n",
       "...            ...          ...                   ...   \n",
       "1343          hold        Boise            735.618312   \n",
       "1344          hold        Boise            709.340206   \n",
       "1345          hold         Kuna            697.672269   \n",
       "1346          hold       Burley            734.242340   \n",
       "1347          hold     Meridian            735.821429   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      740.000000                   740.000000   \n",
       "1                      720.888889                   690.000000   \n",
       "2                      702.359281                   666.966068   \n",
       "3                      680.871795                   642.602564   \n",
       "4                      690.632716                   689.580247   \n",
       "...                           ...                          ...   \n",
       "1343                   738.469939                   738.399890   \n",
       "1344                   750.000000                   680.000000   \n",
       "1345                   690.000000                   690.000000   \n",
       "1346                   746.652925                   699.992201   \n",
       "1347                   740.571429                   739.571429   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     7.0             False         False            False  \n",
       "1                    16.0             False         False            False  \n",
       "2                    10.0              True         False            False  \n",
       "3                    50.0             False         False            False  \n",
       "4                    50.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1343                  0.0             False         False            False  \n",
       "1344                  5.0             False         False            False  \n",
       "1345                  0.0             False         False            False  \n",
       "1346                 60.0              True         False             True  \n",
       "1347                 17.0             False         False            False  \n",
       "\n",
       "[6800 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "ID_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/ID/\" + file)\n",
    "    ID_all = pd.concat([ID_all, df])\n",
    "    \n",
    "ID_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "ID_all.to_csv(\"Scraper_Output/State_Month_Day/ID_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['ID']\n",
      "Unique jan_2018: ['ID']\n",
      "Unique jan_2019: ['ID']\n",
      "Unique jan_2020: ['ID']\n",
      "Unique jan_2021: ['ID']\n",
      "Unique feb_2017: ['ID']\n",
      "Unique feb_2018: ['ID']\n",
      "Unique feb_2019: ['ID']\n",
      "Unique feb_2020: ['ID']\n",
      "Unique feb_2021: ['ID']\n",
      "Unique jun_2017: ['ID']\n",
      "Unique jun_2018: ['ID']\n",
      "Unique jun_2019: ['ID']\n",
      "Unique jun_2020: ['ID']\n",
      "Unique jun_2021: ['ID']\n",
      "Unique jul_2017: ['ID']\n",
      "Unique jul_2018: ['ID']\n",
      "Unique jul_2019: ['ID']\n",
      "Unique jul_2020: ['ID']\n",
      "Unique jul_2021: ['ID']\n",
      "Unique aug_2017: ['ID']\n",
      "Unique aug_2018: ['ID']\n",
      "Unique aug_2019: ['ID']\n",
      "Unique aug_2020: ['ID']\n",
      "Unique dec_2017: ['ID']\n",
      "Unique dec_2018: ['ID']\n",
      "Unique dec_2019: ['ID']\n",
      "Unique dec_2020: ['ID']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mIDe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}