DYD_AZ-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/AZ-day/2017-jan-day-AZ.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c83f8f51b0f8fe74fa93197aa4131490767745d6</td>\n",
       "      <td>2017-01-07 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Glendale</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3a5912e1270f82a855f99429304913f732e2a4d3</td>\n",
       "      <td>2017-01-01 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>728</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9e01bad8e6eb04428a88b5b0980df117fd385100</td>\n",
       "      <td>2017-01-14 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>720</td>\n",
       "      <td>670</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>823fb61f1e5d46890e35b630beb3389bcbe1f0fa</td>\n",
       "      <td>2017-01-10 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Bisbee</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19aa51d82a1e843b5aff3b341dda54e40d11ccdd</td>\n",
       "      <td>2017-01-25 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Litchfield Park</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316207</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-01-09 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316208</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-01-17 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316209</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-01-09 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316210</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-01-17 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>644</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316211</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-01-05 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>649</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>309050 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c83f8f51b0f8fe74fa93197aa4131490767745d6  2017-01-07 17:20:00 UTC   \n",
       "1       3a5912e1270f82a855f99429304913f732e2a4d3  2017-01-01 17:05:00 UTC   \n",
       "2       9e01bad8e6eb04428a88b5b0980df117fd385100  2017-01-14 19:40:00 UTC   \n",
       "3       823fb61f1e5d46890e35b630beb3389bcbe1f0fa  2017-01-10 17:20:00 UTC   \n",
       "4       19aa51d82a1e843b5aff3b341dda54e40d11ccdd  2017-01-25 13:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "316207  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-01-09 14:35:00 UTC   \n",
       "316208  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-01-17 13:30:00 UTC   \n",
       "316209  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-01-09 17:40:00 UTC   \n",
       "316210  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-01-17 15:45:00 UTC   \n",
       "316211  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-01-05 13:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               746                      740   \n",
       "1          auto          auto               728                      765   \n",
       "2          heat          auto               706                      720   \n",
       "3          heat          auto               725                      740   \n",
       "4          heat          hold               699                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "316207     cool          hold               668                      770   \n",
       "316208     cool          hold               652                      770   \n",
       "316209     cool          hold               664                      770   \n",
       "316210     cool          hold               644                      770   \n",
       "316211     cool          hold               649                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                           740            AZ         Glendale   \n",
       "1                           705            AZ           Tucson   \n",
       "2                           670            AZ          Gilbert   \n",
       "3                           740            AZ           Bisbee   \n",
       "4                           700            AZ  Litchfield Park   \n",
       "...                         ...           ...              ...   \n",
       "316207                      770            AZ          Phoenix   \n",
       "316208                      770            AZ          Phoenix   \n",
       "316209                      770            AZ          Phoenix   \n",
       "316210                      770            AZ          Phoenix   \n",
       "316211                      770            AZ          Phoenix   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        15              True         False            False   \n",
       "2                        16             False         False            False   \n",
       "3                        10             False         False            False   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "316207                   65             False         False            False   \n",
       "316208                   65             False         False            False   \n",
       "316209                   65             False         False            False   \n",
       "316210                   65             False         False            False   \n",
       "316211                   65             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "316207                       Gas  \n",
       "316208                       Gas  \n",
       "316209                       Gas  \n",
       "316210                       Gas  \n",
       "316211                       Gas  \n",
       "\n",
       "[309050 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">00abe73a91e4a0012069878eba34b166551cce4d</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Tucson</th>\n",
       "      <td>698.434109</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Tucson</th>\n",
       "      <td>698.580952</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">015b901f68620d06275e3b7aa91aaf7712578c24</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Mesa</th>\n",
       "      <td>699.178571</td>\n",
       "      <td>754.714286</td>\n",
       "      <td>704.571429</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Mesa</th>\n",
       "      <td>693.359813</td>\n",
       "      <td>748.913551</td>\n",
       "      <td>690.128505</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Mesa</th>\n",
       "      <td>688.200000</td>\n",
       "      <td>714.400000</td>\n",
       "      <td>707.200000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Gilbert</th>\n",
       "      <td>701.000000</td>\n",
       "      <td>727.222222</td>\n",
       "      <td>723.888889</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ff3e086b5016e9175c0025a60b8f6db90bb1ef6e</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Fort Mohave</th>\n",
       "      <td>713.241546</td>\n",
       "      <td>783.603865</td>\n",
       "      <td>714.681159</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Fort Mohave</th>\n",
       "      <td>714.615385</td>\n",
       "      <td>786.779487</td>\n",
       "      <td>716.928205</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ffa43b7f70a57be13cf84bc8869b8484ca00b020</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Chandler</th>\n",
       "      <td>676.794118</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>679.705882</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Chandler</th>\n",
       "      <td>689.714286</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>679 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                 698.434109   \n",
       "                                                             hold          Tucson                 698.580952   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                   699.178571   \n",
       "                                                             hold          Mesa                   693.359813   \n",
       "                                                    heat     hold          Mesa                   688.200000   \n",
       "...                                                                                                      ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                701.000000   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave            713.241546   \n",
       "                                                             hold          Fort Mohave            714.615385   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler               676.794118   \n",
       "                                                             hold          Chandler               689.714286   \n",
       "\n",
       "                                                                                        TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                        770.000000   \n",
       "                                                             hold          Tucson                        770.000000   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                          754.714286   \n",
       "                                                             hold          Mesa                          748.913551   \n",
       "                                                    heat     hold          Mesa                          714.400000   \n",
       "...                                                                                                             ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                       727.222222   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave                   783.603865   \n",
       "                                                             hold          Fort Mohave                   786.779487   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler                      770.000000   \n",
       "                                                             hold          Chandler                      770.000000   \n",
       "\n",
       "                                                                                        TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                        700.000000   \n",
       "                                                             hold          Tucson                        700.000000   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                          704.571429   \n",
       "                                                             hold          Mesa                          690.128505   \n",
       "                                                    heat     hold          Mesa                          707.200000   \n",
       "...                                                                                                             ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                       723.888889   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave                   714.681159   \n",
       "                                                             hold          Fort Mohave                   716.928205   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler                      679.705882   \n",
       "                                                             hold          Chandler                      670.000000   \n",
       "\n",
       "                                                                                        Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                               \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                       0.0   \n",
       "                                                             hold          Tucson                       0.0   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                        10.0   \n",
       "                                                             hold          Mesa                        10.0   \n",
       "                                                    heat     hold          Mesa                        10.0   \n",
       "...                                                                                                     ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                      6.0   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave                 25.0   \n",
       "                                                             hold          Fort Mohave                 25.0   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler                    20.0   \n",
       "                                                             hold          Chandler                    20.0   \n",
       "\n",
       "                                                                                        allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                  False   \n",
       "                                                             hold          Tucson                  False   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                    False   \n",
       "                                                             hold          Mesa                    False   \n",
       "                                                    heat     hold          Mesa                    False   \n",
       "...                                                                                                  ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                 False   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave             False   \n",
       "                                                             hold          Fort Mohave             False   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler                False   \n",
       "                                                             hold          Chandler                False   \n",
       "\n",
       "                                                                                        Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                        \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson              False   \n",
       "                                                             hold          Tucson              False   \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                False   \n",
       "                                                             hold          Mesa                False   \n",
       "                                                    heat     hold          Mesa                False   \n",
       "...                                                                                              ...   \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert             False   \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave         False   \n",
       "                                                             hold          Fort Mohave         False   \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler            False   \n",
       "                                                             hold          Chandler            False   \n",
       "\n",
       "                                                                                        Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "00abe73a91e4a0012069878eba34b166551cce4d Jan   2017 auto     auto          Tucson                  True  \n",
       "                                                             hold          Tucson                  True  \n",
       "015b901f68620d06275e3b7aa91aaf7712578c24 Jan   2017 auto     auto          Mesa                   False  \n",
       "                                                             hold          Mesa                   False  \n",
       "                                                    heat     hold          Mesa                   False  \n",
       "...                                                                                                 ...  \n",
       "fe6c9ebd5f40b39cd1ca42027621edd93ac11ec1 Jan   2017 heat     hold          Gilbert                False  \n",
       "ff3e086b5016e9175c0025a60b8f6db90bb1ef6e Jan   2017 auto     auto          Fort Mohave            False  \n",
       "                                                             hold          Fort Mohave            False  \n",
       "ffa43b7f70a57be13cf84bc8869b8484ca00b020 Jan   2017 auto     auto          Chandler               False  \n",
       "                                                             hold          Chandler               False  \n",
       "\n",
       "[679 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/AZ/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/AZ-day/2018-jan-day-AZ.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>405c5f6f964f93249a4601c63320ff0a3d45e6df</td>\n",
       "      <td>2018-01-10 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>678</td>\n",
       "      <td>678</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b5fe7a5af6568e6a574f3868c8da0399e656e27d</td>\n",
       "      <td>2018-01-18 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>845</td>\n",
       "      <td>735</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Vail</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ea2ab7cd11af7014ecc7a481972a29a180107dc7</td>\n",
       "      <td>2018-01-12 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9f3d8ac40541cfc2dc20cce41f7cd5c13619a5d6</td>\n",
       "      <td>2018-01-20 08:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>767</td>\n",
       "      <td>717</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>176057f2ea00c27b53afeb2b2c8284f13ce865b9</td>\n",
       "      <td>2018-01-28 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>805</td>\n",
       "      <td>745</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755086</th>\n",
       "      <td>c90b6b66871e2d1835134af14e3d01688d94fd45</td>\n",
       "      <td>2018-01-03 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755087</th>\n",
       "      <td>3c99bb18e329187e7ed68d3f34d80fc2e5200e0d</td>\n",
       "      <td>2018-01-03 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755088</th>\n",
       "      <td>9f3d8ac40541cfc2dc20cce41f7cd5c13619a5d6</td>\n",
       "      <td>2018-01-07 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755089</th>\n",
       "      <td>f6808434082fd5ed72424eaecc77f26daf2b6d2b</td>\n",
       "      <td>2018-01-05 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755090</th>\n",
       "      <td>c90b6b66871e2d1835134af14e3d01688d94fd45</td>\n",
       "      <td>2018-01-04 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>723122 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       405c5f6f964f93249a4601c63320ff0a3d45e6df  2018-01-10 19:50:00 UTC   \n",
       "1       b5fe7a5af6568e6a574f3868c8da0399e656e27d  2018-01-18 17:05:00 UTC   \n",
       "2       ea2ab7cd11af7014ecc7a481972a29a180107dc7  2018-01-12 17:15:00 UTC   \n",
       "3       9f3d8ac40541cfc2dc20cce41f7cd5c13619a5d6  2018-01-20 08:40:00 UTC   \n",
       "4       176057f2ea00c27b53afeb2b2c8284f13ce865b9  2018-01-28 19:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "755086  c90b6b66871e2d1835134af14e3d01688d94fd45  2018-01-03 16:45:00 UTC   \n",
       "755087  3c99bb18e329187e7ed68d3f34d80fc2e5200e0d  2018-01-03 17:30:00 UTC   \n",
       "755088  9f3d8ac40541cfc2dc20cce41f7cd5c13619a5d6  2018-01-07 17:55:00 UTC   \n",
       "755089  f6808434082fd5ed72424eaecc77f26daf2b6d2b  2018-01-05 15:35:00 UTC   \n",
       "755090  c90b6b66871e2d1835134af14e3d01688d94fd45  2018-01-04 13:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               717                      678   \n",
       "1          auto          hold               739                      845   \n",
       "2          heat          hold               670                      665   \n",
       "3          auto          hold               715                      767   \n",
       "4          auto          hold               745                      805   \n",
       "...         ...           ...               ...                      ...   \n",
       "755086     auto          auto               711                      765   \n",
       "755087     auto          hold               723                      765   \n",
       "755088     auto          hold               740                      765   \n",
       "755089     auto          hold               719                      765   \n",
       "755090     auto          auto               724                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           678            AZ    Phoenix                    0   \n",
       "1                           735            AZ       Vail                   15   \n",
       "2                           665            AZ     Tucson                    0   \n",
       "3                           717            AZ     Tucson                   10   \n",
       "4                           745            AZ       Yuma                   10   \n",
       "...                         ...           ...        ...                  ...   \n",
       "755086                      715            AZ  Sahuarita                   10   \n",
       "755087                      715            AZ   Surprise                    6   \n",
       "755088                      715            AZ     Tucson                   10   \n",
       "755089                      715            AZ    Phoenix                   17   \n",
       "755090                      715            AZ  Sahuarita                   10   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                   True         False             True   \n",
       "4                  False         False             True   \n",
       "...                  ...           ...              ...   \n",
       "755086             False         False            False   \n",
       "755087             False         False            False   \n",
       "755088              True         False             True   \n",
       "755089             False         False            False   \n",
       "755090             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "755086                       Gas  \n",
       "755087                       Gas  \n",
       "755088                  Electric  \n",
       "755089                       Gas  \n",
       "755090                       Gas  \n",
       "\n",
       "[723122 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/AZ/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/AZ-day/2019-jan-day-AZ.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d89ff6104d444090519e43333b6399bcd935e39b</td>\n",
       "      <td>2019-01-11 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5be6f27587f802730ce59c504b6667f7f043b223</td>\n",
       "      <td>2019-01-22 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>92ddb2b34ea659d91151d9f1a781b7132949706a</td>\n",
       "      <td>2019-01-30 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4ba3c60ec596a37b6feed8e6a7b0300baea5ef93</td>\n",
       "      <td>2019-01-09 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Queen Creek</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f0dcccda6e9fe0b963e8d2b9997663ed45ddb034</td>\n",
       "      <td>2019-01-14 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>747</td>\n",
       "      <td>796</td>\n",
       "      <td>746</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Litchfield Park</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148230</th>\n",
       "      <td>7192d7b69031b9c712d7b1da9dd7dc648ef4bd17</td>\n",
       "      <td>2019-01-30 07:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148231</th>\n",
       "      <td>7192d7b69031b9c712d7b1da9dd7dc648ef4bd17</td>\n",
       "      <td>2019-01-16 09:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148232</th>\n",
       "      <td>1b941760cdf1a4e0d4b887a71a6fa78c86796db0</td>\n",
       "      <td>2019-01-04 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Oro Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148234</th>\n",
       "      <td>7ecc29385f3a555f16f7497deb3971a4364f4cdd</td>\n",
       "      <td>2019-01-10 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1148235</th>\n",
       "      <td>803b6a65e173255e6e53174e5be072c453e1a09b</td>\n",
       "      <td>2019-01-23 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>651</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1111214 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        d89ff6104d444090519e43333b6399bcd935e39b  2019-01-11 14:40:00 UTC   \n",
       "1        5be6f27587f802730ce59c504b6667f7f043b223  2019-01-22 17:25:00 UTC   \n",
       "2        92ddb2b34ea659d91151d9f1a781b7132949706a  2019-01-30 19:50:00 UTC   \n",
       "3        4ba3c60ec596a37b6feed8e6a7b0300baea5ef93  2019-01-09 15:15:00 UTC   \n",
       "4        f0dcccda6e9fe0b963e8d2b9997663ed45ddb034  2019-01-14 14:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1148230  7192d7b69031b9c712d7b1da9dd7dc648ef4bd17  2019-01-30 07:35:00 UTC   \n",
       "1148231  7192d7b69031b9c712d7b1da9dd7dc648ef4bd17  2019-01-16 09:20:00 UTC   \n",
       "1148232  1b941760cdf1a4e0d4b887a71a6fa78c86796db0  2019-01-04 15:55:00 UTC   \n",
       "1148234  7ecc29385f3a555f16f7497deb3971a4364f4cdd  2019-01-10 17:25:00 UTC   \n",
       "1148235  803b6a65e173255e6e53174e5be072c453e1a09b  2019-01-23 17:25:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               717                      765   \n",
       "1           heat          hold               685                      685   \n",
       "2           heat          hold               707                      702   \n",
       "3           heat          hold               690                      655   \n",
       "4           auto          auto               747                      796   \n",
       "...          ...           ...               ...                      ...   \n",
       "1148230     auto          hold               732                      760   \n",
       "1148231     auto          hold               723                      760   \n",
       "1148232     heat          hold               757                      760   \n",
       "1148234     heat          auto               696                      760   \n",
       "1148235     auto          hold               651                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            715            AZ           Tucson   \n",
       "1                            685            AZ         Surprise   \n",
       "2                            702            AZ             Mesa   \n",
       "3                            655            AZ      Queen Creek   \n",
       "4                            746            AZ  Litchfield Park   \n",
       "...                          ...           ...              ...   \n",
       "1148230                      710            AZ          Phoenix   \n",
       "1148231                      710            AZ          Phoenix   \n",
       "1148232                      760            AZ       Oro Valley   \n",
       "1148234                      700            AZ         Surprise   \n",
       "1148235                      620            AZ          Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         15             False         False             True   \n",
       "1                         20              True         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         16             False         False             True   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1148230                   20             False         False             True   \n",
       "1148231                   20             False         False             True   \n",
       "1148232                   10              True         False             True   \n",
       "1148234                   10             False         False            False   \n",
       "1148235                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1148230                  Electric  \n",
       "1148231                  Electric  \n",
       "1148232                  Electric  \n",
       "1148234                       Gas  \n",
       "1148235                       Gas  \n",
       "\n",
       "[1111214 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/AZ/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/AZ-day/2020-jan-day-AZ.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ac3cd242daf926d1590edd256c1d9a02e4d8f3e7</td>\n",
       "      <td>2020-01-11 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a4ad8ff8b6a0bd64323da29f6de3ab5bba31f91</td>\n",
       "      <td>2020-01-17 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>776</td>\n",
       "      <td>726</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Thatcher</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5ef65b3d4847473b122c9e91f5ae4aa4a840879e</td>\n",
       "      <td>2020-01-15 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0951b1fd22270363781e81814be00d51bc77124d</td>\n",
       "      <td>2020-01-19 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0f2432d3a759c78ce4932d9da08a3eb6201e7d35</td>\n",
       "      <td>2020-01-19 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413795</th>\n",
       "      <td>98d6abafcccd37d627e0eb12a6d952b6f0eb4310</td>\n",
       "      <td>2020-01-28 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413796</th>\n",
       "      <td>715de17e9d8f355dfc6677579276748e2c3e02f4</td>\n",
       "      <td>2020-01-01 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sun City</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413797</th>\n",
       "      <td>be5ec3fb4f5d19984380a06a56054eb963dde34f</td>\n",
       "      <td>2020-01-12 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Litchfield Park</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413798</th>\n",
       "      <td>2af6100a8a96b071a6f947555ece78e6295cd626</td>\n",
       "      <td>2020-01-21 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1413799</th>\n",
       "      <td>14cd51c2bc951dda1dbe35c3d74852170e0b0ebe</td>\n",
       "      <td>2020-01-31 09:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1350679 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        ac3cd242daf926d1590edd256c1d9a02e4d8f3e7  2020-01-11 12:40:00 UTC   \n",
       "1        5a4ad8ff8b6a0bd64323da29f6de3ab5bba31f91  2020-01-17 14:50:00 UTC   \n",
       "2        5ef65b3d4847473b122c9e91f5ae4aa4a840879e  2020-01-15 19:30:00 UTC   \n",
       "4        0951b1fd22270363781e81814be00d51bc77124d  2020-01-19 15:20:00 UTC   \n",
       "5        0f2432d3a759c78ce4932d9da08a3eb6201e7d35  2020-01-19 18:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1413795  98d6abafcccd37d627e0eb12a6d952b6f0eb4310  2020-01-28 15:45:00 UTC   \n",
       "1413796  715de17e9d8f355dfc6677579276748e2c3e02f4  2020-01-01 19:25:00 UTC   \n",
       "1413797  be5ec3fb4f5d19984380a06a56054eb963dde34f  2020-01-12 13:40:00 UTC   \n",
       "1413798  2af6100a8a96b071a6f947555ece78e6295cd626  2020-01-21 19:00:00 UTC   \n",
       "1413799  14cd51c2bc951dda1dbe35c3d74852170e0b0ebe  2020-01-31 09:20:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               680                      705   \n",
       "1           auto          hold               725                      776   \n",
       "2           heat          hold               673                      665   \n",
       "4           auto          auto               724                      765   \n",
       "5           heat          hold               717                      685   \n",
       "...          ...           ...               ...                      ...   \n",
       "1413795     heat          hold               706                      760   \n",
       "1413796     auto          hold               712                      760   \n",
       "1413797     heat          hold               760                      760   \n",
       "1413798     auto          hold               676                      760   \n",
       "1413799     heat          auto               757                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            705            AZ         Chandler   \n",
       "1                            726            AZ         Thatcher   \n",
       "2                            665            AZ             Mesa   \n",
       "4                            715            AZ           Tucson   \n",
       "5                            685            AZ          Phoenix   \n",
       "...                          ...           ...              ...   \n",
       "1413795                      710            AZ           Tucson   \n",
       "1413796                      710            AZ         Sun City   \n",
       "1413797                      760            AZ  Litchfield Park   \n",
       "1413798                      660            AZ          Phoenix   \n",
       "1413799                      760            AZ          Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "1                          5              True         False             True   \n",
       "2                          5             False         False            False   \n",
       "4                         29             False         False            False   \n",
       "5                         25             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1413795                    7             False         False            False   \n",
       "1413796                   47             False         False             True   \n",
       "1413797                    0             False         False            False   \n",
       "1413798                   27             False         False             True   \n",
       "1413799                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "4                             Gas  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "1413795                       Gas  \n",
       "1413796                  Electric  \n",
       "1413797                       Gas  \n",
       "1413798                  Electric  \n",
       "1413799                       Gas  \n",
       "\n",
       "[1350679 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/AZ/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/AZ-day/2021-jan-day-AZ.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8c617741c974ea3d95df96ae55f9f86aaab35a26</td>\n",
       "      <td>2021-01-27 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d7c45ac5a0360fe7053d9b1883c9fd31862522f6</td>\n",
       "      <td>2021-01-23 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>787</td>\n",
       "      <td>727</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>71baa5afa559fe1dbce98c262bef377272fdabe2</td>\n",
       "      <td>2021-01-12 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Flagstaff</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f1f20b2ba0614b999eb9d8dd4c979dfea299bdc9</td>\n",
       "      <td>2021-01-03 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>767</td>\n",
       "      <td>697</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7319ce6572048ac74ef266de139f4576e474249f</td>\n",
       "      <td>2021-01-04 08:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874904</th>\n",
       "      <td>52e92d3957712db8fd805073e024c90a95817381</td>\n",
       "      <td>2021-01-13 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874905</th>\n",
       "      <td>0ac042f6fc949edceac5fb1ecda5a16bc4e45b77</td>\n",
       "      <td>2021-01-13 08:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>stanfield</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874906</th>\n",
       "      <td>f84367a74e71563922b6830678b76cce513f8ac1</td>\n",
       "      <td>2021-01-02 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Avondale</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874907</th>\n",
       "      <td>0ac042f6fc949edceac5fb1ecda5a16bc4e45b77</td>\n",
       "      <td>2021-01-12 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>stanfield</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>874908</th>\n",
       "      <td>5e71e3d82d7f5a6c0fe916524116fa3e36de41b5</td>\n",
       "      <td>2021-01-09 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>832536 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8c617741c974ea3d95df96ae55f9f86aaab35a26  2021-01-27 15:05:00 UTC   \n",
       "1       d7c45ac5a0360fe7053d9b1883c9fd31862522f6  2021-01-23 16:05:00 UTC   \n",
       "2       71baa5afa559fe1dbce98c262bef377272fdabe2  2021-01-12 13:55:00 UTC   \n",
       "3       f1f20b2ba0614b999eb9d8dd4c979dfea299bdc9  2021-01-03 15:20:00 UTC   \n",
       "4       7319ce6572048ac74ef266de139f4576e474249f  2021-01-04 08:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "874904  52e92d3957712db8fd805073e024c90a95817381  2021-01-13 14:35:00 UTC   \n",
       "874905  0ac042f6fc949edceac5fb1ecda5a16bc4e45b77  2021-01-13 08:30:00 UTC   \n",
       "874906  f84367a74e71563922b6830678b76cce513f8ac1  2021-01-02 19:35:00 UTC   \n",
       "874907  0ac042f6fc949edceac5fb1ecda5a16bc4e45b77  2021-01-12 17:45:00 UTC   \n",
       "874908  5e71e3d82d7f5a6c0fe916524116fa3e36de41b5  2021-01-09 19:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               671                      685   \n",
       "1          auto          hold               727                      787   \n",
       "2          heat          hold               657                      675   \n",
       "3          auto          hold               684                      767   \n",
       "4          heat          hold               696                      699   \n",
       "...         ...           ...               ...                      ...   \n",
       "874904     auto          hold               694                      765   \n",
       "874905     heat          hold               757                      765   \n",
       "874906     auto          hold               711                      765   \n",
       "874907     heat          hold               765                      765   \n",
       "874908     auto          hold               712                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           685            AZ     Tucson                   25   \n",
       "1                           727            AZ      Tempe                    0   \n",
       "2                           675            AZ  Flagstaff                   30   \n",
       "3                           697            AZ    Phoenix                   10   \n",
       "4                           699            AZ     Tucson                   60   \n",
       "...                         ...           ...        ...                  ...   \n",
       "874904                      705            AZ    Phoenix                   50   \n",
       "874905                      765            AZ  stanfield                   10   \n",
       "874906                      715            AZ   Avondale                   20   \n",
       "874907                      765            AZ  stanfield                   10   \n",
       "874908                      715            AZ       Mesa                   30   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False             True   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                   True         False            False   \n",
       "...                  ...           ...              ...   \n",
       "874904             False         False             True   \n",
       "874905              True         False             True   \n",
       "874906             False         False            False   \n",
       "874907              True         False             True   \n",
       "874908             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "874904                  Electric  \n",
       "874905                  Electric  \n",
       "874906                       Gas  \n",
       "874907                  Electric  \n",
       "874908                  Electric  \n",
       "\n",
       "[832536 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/AZ/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>698.434109</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>698.580952</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>015b901f68620d06275e3b7aa91aaf7712578c24</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>699.178571</td>\n",
       "      <td>754.714286</td>\n",
       "      <td>704.571429</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>015b901f68620d06275e3b7aa91aaf7712578c24</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>693.359813</td>\n",
       "      <td>748.913551</td>\n",
       "      <td>690.128505</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>015b901f68620d06275e3b7aa91aaf7712578c24</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>688.200000</td>\n",
       "      <td>714.400000</td>\n",
       "      <td>707.200000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1311</th>\n",
       "      <td>ff3e086b5016e9175c0025a60b8f6db90bb1ef6e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Mohave</td>\n",
       "      <td>690.630137</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>676.438356</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1312</th>\n",
       "      <td>ffc69aa005f871624e7187d94b86f1d795252fd7</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>690.871212</td>\n",
       "      <td>695.078283</td>\n",
       "      <td>695.078283</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>693.427263</td>\n",
       "      <td>799.582909</td>\n",
       "      <td>692.868769</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1314</th>\n",
       "      <td>fffe6c24cd5a508c58628472cd5bd31846ea70e4</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>701.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>fffe6c24cd5a508c58628472cd5bd31846ea70e4</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>696.250000</td>\n",
       "      <td>662.000000</td>\n",
       "      <td>659.250000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8033 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00abe73a91e4a0012069878eba34b166551cce4d   Jan  2017     auto   \n",
       "1     00abe73a91e4a0012069878eba34b166551cce4d   Jan  2017     auto   \n",
       "2     015b901f68620d06275e3b7aa91aaf7712578c24   Jan  2017     auto   \n",
       "3     015b901f68620d06275e3b7aa91aaf7712578c24   Jan  2017     auto   \n",
       "4     015b901f68620d06275e3b7aa91aaf7712578c24   Jan  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1311  ff3e086b5016e9175c0025a60b8f6db90bb1ef6e   Jan  2021     auto   \n",
       "1312  ffc69aa005f871624e7187d94b86f1d795252fd7   Jan  2021     heat   \n",
       "1313  ffec90658ad5a300da53b43a28fe991808fb4006   Jan  2021     auto   \n",
       "1314  fffe6c24cd5a508c58628472cd5bd31846ea70e4   Jan  2021     cool   \n",
       "1315  fffe6c24cd5a508c58628472cd5bd31846ea70e4   Jan  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto        Tucson            698.434109   \n",
       "1             hold        Tucson            698.580952   \n",
       "2             auto          Mesa            699.178571   \n",
       "3             hold          Mesa            693.359813   \n",
       "4             hold          Mesa            688.200000   \n",
       "...            ...           ...                   ...   \n",
       "1311          hold   Fort Mohave            690.630137   \n",
       "1312          hold       Phoenix            690.871212   \n",
       "1313          hold  Sierra Vista            693.427263   \n",
       "1314          hold       Phoenix            701.000000   \n",
       "1315          hold       Phoenix            696.250000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      770.000000                   700.000000   \n",
       "1                      770.000000                   700.000000   \n",
       "2                      754.714286                   704.571429   \n",
       "3                      748.913551                   690.128505   \n",
       "4                      714.400000                   707.200000   \n",
       "...                           ...                          ...   \n",
       "1311                   780.000000                   676.438356   \n",
       "1312                   695.078283                   695.078283   \n",
       "1313                   799.582909                   692.868769   \n",
       "1314                   650.000000                   650.000000   \n",
       "1315                   662.000000                   659.250000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False             True  \n",
       "1                     0.0             False         False             True  \n",
       "2                    10.0             False         False            False  \n",
       "3                    10.0             False         False            False  \n",
       "4                    10.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1311                 25.0             False         False            False  \n",
       "1312                 50.0              True         False             True  \n",
       "1313                  0.0             False         False            False  \n",
       "1314                 10.0              True         False            False  \n",
       "1315                 10.0              True         False            False  \n",
       "\n",
       "[8033 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/jan/\" + file)\n",
    "    AZ_jan = pd.concat([AZ_jan, df])\n",
    "    \n",
    "AZ_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_jan.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/AZ-day/2017-feb-day-AZ.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>664aaf4176ce2239ff5bc658ab25fe45f2ddd19e</td>\n",
       "      <td>2017-02-26 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>679</td>\n",
       "      <td>680</td>\n",
       "      <td>690</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0cced87df237140f4072fcd7d1576422d2dbbb0a</td>\n",
       "      <td>2017-02-02 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>668</td>\n",
       "      <td>820</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c83f8f51b0f8fe74fa93197aa4131490767745d6</td>\n",
       "      <td>2017-02-03 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Glendale</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>747c1d901235ece585427c3e4b4b6aed55ee3913</td>\n",
       "      <td>2017-02-05 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>720</td>\n",
       "      <td>660</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Oro Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9b22d3a2ff9a7c03d7ddec0f6ebad9928466b325</td>\n",
       "      <td>2017-02-25 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>771</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Arizona City</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272425</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-02-02 19:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>679</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272426</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-02-22 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272427</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-02-05 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>652</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272428</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-02-12 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>272429</th>\n",
       "      <td>f4204b8ee90aa8930df52207631d03ec09b1f1a4</td>\n",
       "      <td>2017-02-25 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>647</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>264700 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       664aaf4176ce2239ff5bc658ab25fe45f2ddd19e  2017-02-26 18:45:00 UTC   \n",
       "1       0cced87df237140f4072fcd7d1576422d2dbbb0a  2017-02-02 16:55:00 UTC   \n",
       "2       c83f8f51b0f8fe74fa93197aa4131490767745d6  2017-02-03 15:15:00 UTC   \n",
       "3       747c1d901235ece585427c3e4b4b6aed55ee3913  2017-02-05 17:25:00 UTC   \n",
       "4       9b22d3a2ff9a7c03d7ddec0f6ebad9928466b325  2017-02-25 16:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "272425  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-02-02 19:45:00 UTC   \n",
       "272426  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-02-22 14:25:00 UTC   \n",
       "272427  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-02-05 18:15:00 UTC   \n",
       "272428  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-02-12 14:35:00 UTC   \n",
       "272429  f4204b8ee90aa8930df52207631d03ec09b1f1a4  2017-02-25 17:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               679                      680   \n",
       "1          auto          auto               668                      820   \n",
       "2          heat          hold               743                      750   \n",
       "3          heat          auto               673                      720   \n",
       "4          heat          auto               771                      770   \n",
       "...         ...           ...               ...                      ...   \n",
       "272425     heat          auto               679                      650   \n",
       "272426     heat          auto               710                      650   \n",
       "272427     heat          auto               652                      650   \n",
       "272428     heat          auto               723                      650   \n",
       "272429     heat          auto               647                      650   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           690            AZ      Chandler   \n",
       "1                           640            AZ        Tucson   \n",
       "2                           750            AZ      Glendale   \n",
       "3                           660            AZ    Oro Valley   \n",
       "4                           770            AZ  Arizona City   \n",
       "...                         ...           ...           ...   \n",
       "272425                      650            AZ       Phoenix   \n",
       "272426                      650            AZ       Phoenix   \n",
       "272427                      650            AZ       Phoenix   \n",
       "272428                      650            AZ       Phoenix   \n",
       "272429                      650            AZ       Phoenix   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False             True   \n",
       "1                        10             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                         5             False         False             True   \n",
       "4                        15             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "272425                   65             False         False            False   \n",
       "272426                   65             False         False            False   \n",
       "272427                   65             False         False            False   \n",
       "272428                   65             False         False            False   \n",
       "272429                   65             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "272425                       Gas  \n",
       "272426                       Gas  \n",
       "272427                       Gas  \n",
       "272428                       Gas  \n",
       "272429                       Gas  \n",
       "\n",
       "[264700 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/AZ/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/AZ-day/2018-feb-day-AZ.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>af019b5d786eb9ad785963329fa216232f30d30e</td>\n",
       "      <td>2018-02-14 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>788</td>\n",
       "      <td>662</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>26</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1d4dfb7ff62cc2996605aea93c20797b1dcd717c</td>\n",
       "      <td>2018-02-26 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f5b7e1ec79db6e4430197f9988e2b0b9d6f81347</td>\n",
       "      <td>2018-02-18 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>685</td>\n",
       "      <td>718</td>\n",
       "      <td>730</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4f0fefd7929a9b1a58368fd50b62e01c0ca7fc8a</td>\n",
       "      <td>2018-02-11 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>830</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sun City</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e1505b480623eb6cca8b3491435fc4e931085c97</td>\n",
       "      <td>2018-02-08 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683460</th>\n",
       "      <td>5cb96df182e0fb88438744dadaf141e4748ab3b4</td>\n",
       "      <td>2018-02-04 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Thatcher</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683461</th>\n",
       "      <td>2623afd8550dd9ae290a7d6d2f63179d36ecdc9a</td>\n",
       "      <td>2018-02-09 07:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683462</th>\n",
       "      <td>2623afd8550dd9ae290a7d6d2f63179d36ecdc9a</td>\n",
       "      <td>2018-02-12 08:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683463</th>\n",
       "      <td>5cb96df182e0fb88438744dadaf141e4748ab3b4</td>\n",
       "      <td>2018-02-03 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Thatcher</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683464</th>\n",
       "      <td>5cb96df182e0fb88438744dadaf141e4748ab3b4</td>\n",
       "      <td>2018-02-23 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Thatcher</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>657068 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       af019b5d786eb9ad785963329fa216232f30d30e  2018-02-14 15:15:00 UTC   \n",
       "2       1d4dfb7ff62cc2996605aea93c20797b1dcd717c  2018-02-26 16:00:00 UTC   \n",
       "3       f5b7e1ec79db6e4430197f9988e2b0b9d6f81347  2018-02-18 15:10:00 UTC   \n",
       "4       4f0fefd7929a9b1a58368fd50b62e01c0ca7fc8a  2018-02-11 18:45:00 UTC   \n",
       "5       e1505b480623eb6cca8b3491435fc4e931085c97  2018-02-08 19:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "683460  5cb96df182e0fb88438744dadaf141e4748ab3b4  2018-02-04 18:30:00 UTC   \n",
       "683461  2623afd8550dd9ae290a7d6d2f63179d36ecdc9a  2018-02-09 07:35:00 UTC   \n",
       "683462  2623afd8550dd9ae290a7d6d2f63179d36ecdc9a  2018-02-12 08:30:00 UTC   \n",
       "683463  5cb96df182e0fb88438744dadaf141e4748ab3b4  2018-02-03 15:00:00 UTC   \n",
       "683464  5cb96df182e0fb88438744dadaf141e4748ab3b4  2018-02-23 14:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               690                      788   \n",
       "2          heat          hold               679                      677   \n",
       "3          cool          auto               685                      718   \n",
       "4          heat          auto               716                      830   \n",
       "5          heat          auto               696                      725   \n",
       "...         ...           ...               ...                      ...   \n",
       "683460     auto          hold               717                      765   \n",
       "683461     auto          auto               739                      765   \n",
       "683462     auto          auto               743                      765   \n",
       "683463     auto          hold               709                      765   \n",
       "683464     auto          hold               712                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           662            AZ     Phoenix   \n",
       "2                           677            AZ     Phoenix   \n",
       "3                           730            AZ  Cave Creek   \n",
       "4                           640            AZ    Sun City   \n",
       "5                           675            AZ     Phoenix   \n",
       "...                         ...           ...         ...   \n",
       "683460                      715            AZ    Thatcher   \n",
       "683461                      715            AZ      Tucson   \n",
       "683462                      715            AZ      Tucson   \n",
       "683463                      715            AZ    Thatcher   \n",
       "683464                      715            AZ    Thatcher   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        26             False         False            False   \n",
       "2                        57             False         False             True   \n",
       "3                        20             False         False            False   \n",
       "4                        47             False         False             True   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "683460                   10             False         False            False   \n",
       "683461                    0             False         False             True   \n",
       "683462                    0             False         False             True   \n",
       "683463                   10             False         False            False   \n",
       "683464                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "683460                       Gas  \n",
       "683461                  Electric  \n",
       "683462                  Electric  \n",
       "683463                       Gas  \n",
       "683464                       Gas  \n",
       "\n",
       "[657068 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/AZ/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/AZ-day/2019-feb-day-AZ.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a95bce92e10a8a6857ff5f060c831d60e42c85a1</td>\n",
       "      <td>2019-02-25 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c8c04c2688597b5133a0d02e6007382b9cdd6b95</td>\n",
       "      <td>2019-02-12 07:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>785</td>\n",
       "      <td>695</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gold Canyon</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1d4dfb7ff62cc2996605aea93c20797b1dcd717c</td>\n",
       "      <td>2019-02-22 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>686</td>\n",
       "      <td>706</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>126f9f43fc99603d36dfedbdc0cbe8632e455d45</td>\n",
       "      <td>2019-02-19 09:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>772</td>\n",
       "      <td>702</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b536e23aa86390a609b6d268dbcb2c5e24b38d90</td>\n",
       "      <td>2019-02-24 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>712</td>\n",
       "      <td>662</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793068</th>\n",
       "      <td>6d49a6eba7c643bfb165271ffded5dcb473e14b1</td>\n",
       "      <td>2019-02-28 10:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793069</th>\n",
       "      <td>d1444dc33e577ebac3f1ef2ebee9f45dc20f5cf0</td>\n",
       "      <td>2019-02-16 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>68</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793070</th>\n",
       "      <td>eaec5f8513536d6aa30949f96572ad6b9b9a1c63</td>\n",
       "      <td>2019-02-21 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793071</th>\n",
       "      <td>7192d7b69031b9c712d7b1da9dd7dc648ef4bd17</td>\n",
       "      <td>2019-02-08 07:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793072</th>\n",
       "      <td>8f12025e7ca02b79c38da7a87fb7c4e0cac669d5</td>\n",
       "      <td>2019-02-21 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>764163 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a95bce92e10a8a6857ff5f060c831d60e42c85a1  2019-02-25 16:55:00 UTC   \n",
       "1       c8c04c2688597b5133a0d02e6007382b9cdd6b95  2019-02-12 07:20:00 UTC   \n",
       "2       1d4dfb7ff62cc2996605aea93c20797b1dcd717c  2019-02-22 13:30:00 UTC   \n",
       "3       126f9f43fc99603d36dfedbdc0cbe8632e455d45  2019-02-19 09:10:00 UTC   \n",
       "4       b536e23aa86390a609b6d268dbcb2c5e24b38d90  2019-02-24 15:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "793068  6d49a6eba7c643bfb165271ffded5dcb473e14b1  2019-02-28 10:00:00 UTC   \n",
       "793069  d1444dc33e577ebac3f1ef2ebee9f45dc20f5cf0  2019-02-16 13:35:00 UTC   \n",
       "793070  eaec5f8513536d6aa30949f96572ad6b9b9a1c63  2019-02-21 13:35:00 UTC   \n",
       "793071  7192d7b69031b9c712d7b1da9dd7dc648ef4bd17  2019-02-08 07:05:00 UTC   \n",
       "793072  8f12025e7ca02b79c38da7a87fb7c4e0cac669d5  2019-02-21 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               729                      785   \n",
       "1          auto          hold               689                      785   \n",
       "2          heat          auto               720                      686   \n",
       "3          auto          hold               731                      772   \n",
       "4          cool          hold               674                      712   \n",
       "...         ...           ...               ...                      ...   \n",
       "793068     cool          auto               727                      760   \n",
       "793069     auto          hold               677                      760   \n",
       "793070     heat          auto               759                      760   \n",
       "793071     auto          hold               711                      760   \n",
       "793072     heat          auto               693                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           735            AZ     Chandler   \n",
       "1                           695            AZ  Gold Canyon   \n",
       "2                           706            AZ      Phoenix   \n",
       "3                           702            AZ         Yuma   \n",
       "4                           662            AZ        Tempe   \n",
       "...                         ...           ...          ...   \n",
       "793068                      760            AZ         Yuma   \n",
       "793069                      680            AZ       Tucson   \n",
       "793070                      760            AZ       Peoria   \n",
       "793071                      710            AZ      Phoenix   \n",
       "793072                      700            AZ     Chandler   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                        20             False         False             True   \n",
       "2                        57             False         False             True   \n",
       "3                         0             False         False            False   \n",
       "4                         9             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "793068                   15             False         False            False   \n",
       "793069                   68             False         False            False   \n",
       "793070                   45              True         False             True   \n",
       "793071                   20             False         False             True   \n",
       "793072                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "793068                       Gas  \n",
       "793069                       Gas  \n",
       "793070                  Electric  \n",
       "793071                  Electric  \n",
       "793072                       Gas  \n",
       "\n",
       "[764163 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/AZ/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/AZ-day/2020-feb-day-AZ.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a51a1969ab2e263df9c211cc7a8b37e4c0924ee2</td>\n",
       "      <td>2020-02-10 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f77103e2b7e215941f1f250a3286a2802e4013c6</td>\n",
       "      <td>2020-02-03 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1dfc4ad22860756eb43f5e0f505f5d07af63da87</td>\n",
       "      <td>2020-02-04 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>797</td>\n",
       "      <td>701</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ccdbf4852773530bf6e1374bc14d2c1a21155148</td>\n",
       "      <td>2020-02-14 08:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>772</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ccdbf4852773530bf6e1374bc14d2c1a21155148</td>\n",
       "      <td>2020-02-22 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>772</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167653</th>\n",
       "      <td>3117da1ff75835d0ba7de43aebb36cf2f3b2258f</td>\n",
       "      <td>2020-02-11 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167654</th>\n",
       "      <td>1e3f1c09e0ea8c8d5a658c8ffaa7fcf39c00ef26</td>\n",
       "      <td>2020-02-23 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Goodyear</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167655</th>\n",
       "      <td>8864ec0a418dd37a6e729ade9a6c09281b3baa65</td>\n",
       "      <td>2020-02-27 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167656</th>\n",
       "      <td>b02ebaf63ef4c0dc2d44ea0fa2f1950b05246fc6</td>\n",
       "      <td>2020-02-18 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1167657</th>\n",
       "      <td>34f0ff6491df2216a93fa1ddc0d68a82a2285556</td>\n",
       "      <td>2020-02-11 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1105155 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a51a1969ab2e263df9c211cc7a8b37e4c0924ee2  2020-02-10 15:45:00 UTC   \n",
       "1        f77103e2b7e215941f1f250a3286a2802e4013c6  2020-02-03 14:30:00 UTC   \n",
       "2        1dfc4ad22860756eb43f5e0f505f5d07af63da87  2020-02-04 16:40:00 UTC   \n",
       "3        ccdbf4852773530bf6e1374bc14d2c1a21155148  2020-02-14 08:40:00 UTC   \n",
       "4        ccdbf4852773530bf6e1374bc14d2c1a21155148  2020-02-22 18:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1167653  3117da1ff75835d0ba7de43aebb36cf2f3b2258f  2020-02-11 16:30:00 UTC   \n",
       "1167654  1e3f1c09e0ea8c8d5a658c8ffaa7fcf39c00ef26  2020-02-23 17:00:00 UTC   \n",
       "1167655  8864ec0a418dd37a6e729ade9a6c09281b3baa65  2020-02-27 15:30:00 UTC   \n",
       "1167656  b02ebaf63ef4c0dc2d44ea0fa2f1950b05246fc6  2020-02-18 19:10:00 UTC   \n",
       "1167657  34f0ff6491df2216a93fa1ddc0d68a82a2285556  2020-02-11 14:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               724                      725   \n",
       "1           heat          hold               711                      715   \n",
       "2           auto          hold               704                      797   \n",
       "3           heat          auto               692                      772   \n",
       "4           heat          auto               719                      772   \n",
       "...          ...           ...               ...                      ...   \n",
       "1167653     heat          hold               772                      760   \n",
       "1167654     heat          hold               759                      760   \n",
       "1167655     auto          hold               696                      760   \n",
       "1167656     auto          hold               705                      760   \n",
       "1167657     auto          auto               705                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                            725            AZ  Scottsdale   \n",
       "1                            715            AZ     Phoenix   \n",
       "2                            701            AZ        Mesa   \n",
       "3                            700            AZ     Buckeye   \n",
       "4                            700            AZ     Buckeye   \n",
       "...                          ...           ...         ...   \n",
       "1167653                      760            AZ      Peoria   \n",
       "1167654                      760            AZ    Goodyear   \n",
       "1167655                      700            AZ     Phoenix   \n",
       "1167656                      690            AZ    Chandler   \n",
       "1167657                      710            AZ     Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30             False         False            False   \n",
       "1                         69             False         False            False   \n",
       "2                          5             False         False            False   \n",
       "3                          0              True         False             True   \n",
       "4                          0              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1167653                   30             False         False            False   \n",
       "1167654                    7             False         False            False   \n",
       "1167655                    0             False         False             True   \n",
       "1167656                   25             False         False             True   \n",
       "1167657                   19              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1167653                       Gas  \n",
       "1167654                       Gas  \n",
       "1167655                  Electric  \n",
       "1167656                  Electric  \n",
       "1167657                       Gas  \n",
       "\n",
       "[1105155 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/AZ/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/AZ-day/2021-feb-day-AZ.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>442a6c9846193a8f54db23adaea3830054c3c75d</td>\n",
       "      <td>2021-02-27 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>840</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f5fd44aaddb7f2d604ef7ddc97e23187531d6be1</td>\n",
       "      <td>2021-02-19 08:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>04d4e09912e078ee4c318f02c6473aa329d24952</td>\n",
       "      <td>2021-02-08 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>762</td>\n",
       "      <td>636</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Maricopa</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>92303fda727ee4a870cf7b12957de34912ac0b0a</td>\n",
       "      <td>2021-02-09 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>793</td>\n",
       "      <td>743</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f5fd44aaddb7f2d604ef7ddc97e23187531d6be1</td>\n",
       "      <td>2021-02-15 10:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650909</th>\n",
       "      <td>45da57e128203e5f592a15106401255eb70d565d</td>\n",
       "      <td>2021-02-04 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650910</th>\n",
       "      <td>2e03eb06fafd5334563db1275fa5f9fe2391453c</td>\n",
       "      <td>2021-02-13 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650911</th>\n",
       "      <td>ca83f0f00cccfacd19af3be18e0e0a3a7c925fd6</td>\n",
       "      <td>2021-02-19 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650912</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2021-02-10 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650913</th>\n",
       "      <td>d036f67070bfaeeb892add66945a793bbdb9617f</td>\n",
       "      <td>2021-02-04 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>623814 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       442a6c9846193a8f54db23adaea3830054c3c75d  2021-02-27 14:50:00 UTC   \n",
       "1       f5fd44aaddb7f2d604ef7ddc97e23187531d6be1  2021-02-19 08:55:00 UTC   \n",
       "2       04d4e09912e078ee4c318f02c6473aa329d24952  2021-02-08 14:05:00 UTC   \n",
       "3       92303fda727ee4a870cf7b12957de34912ac0b0a  2021-02-09 14:45:00 UTC   \n",
       "4       f5fd44aaddb7f2d604ef7ddc97e23187531d6be1  2021-02-15 10:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "650909  45da57e128203e5f592a15106401255eb70d565d  2021-02-04 18:30:00 UTC   \n",
       "650910  2e03eb06fafd5334563db1275fa5f9fe2391453c  2021-02-13 18:10:00 UTC   \n",
       "650911  ca83f0f00cccfacd19af3be18e0e0a3a7c925fd6  2021-02-19 14:00:00 UTC   \n",
       "650912  80467f30cce2c321e9e05405960ce92a377f38bc  2021-02-10 13:35:00 UTC   \n",
       "650913  d036f67070bfaeeb892add66945a793bbdb9617f  2021-02-04 18:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               756                      840   \n",
       "1          heat          hold               758                      757   \n",
       "2          heat          hold               703                      762   \n",
       "3          auto          hold               741                      793   \n",
       "4          heat          hold               757                      757   \n",
       "...         ...           ...               ...                      ...   \n",
       "650909     heat          hold               758                      760   \n",
       "650910     auto          hold               708                      760   \n",
       "650911     auto          hold               670                      760   \n",
       "650912     heat          hold               757                      760   \n",
       "650913     auto          hold               703                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           760            AZ          Peoria   \n",
       "1                           757            AZ            Yuma   \n",
       "2                           636            AZ        Maricopa   \n",
       "3                           743            AZ      Scottsdale   \n",
       "4                           757            AZ            Yuma   \n",
       "...                         ...           ...             ...   \n",
       "650909                      760            AZ         Buckeye   \n",
       "650910                      710            AZ        Chandler   \n",
       "650911                      690            AZ          Peoria   \n",
       "650912                      760            AZ  San Tan Valley   \n",
       "650913                      700            AZ         Phoenix   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         9              True         False            False   \n",
       "1                         5             False         False             True   \n",
       "2                        10             False         False            False   \n",
       "3                        10             False         False             True   \n",
       "4                         5             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "650909                    7             False         False            False   \n",
       "650910                   20             False         False             True   \n",
       "650911                   30             False         False             True   \n",
       "650912                   10             False         False            False   \n",
       "650913                   70             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "650909                       Gas  \n",
       "650910                  Electric  \n",
       "650911                  Electric  \n",
       "650912                       Gas  \n",
       "650913                  Electric  \n",
       "\n",
       "[623814 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/AZ/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>688.600000</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>015b901f68620d06275e3b7aa91aaf7712578c24</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>698.472727</td>\n",
       "      <td>721.672727</td>\n",
       "      <td>658.400000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01e3ae470e88aac33b2483a97f19afafb926128c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>735.669477</td>\n",
       "      <td>778.387015</td>\n",
       "      <td>727.600337</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01e3ae470e88aac33b2483a97f19afafb926128c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>735.939170</td>\n",
       "      <td>778.408579</td>\n",
       "      <td>727.329114</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0222630f87433cb6a60e0fe83c9bc2233a842fce</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>738.236246</td>\n",
       "      <td>757.640777</td>\n",
       "      <td>753.097087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1127</th>\n",
       "      <td>fe22e9c2412fbf136c42924645795c639968ca3a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>753.611285</td>\n",
       "      <td>754.023511</td>\n",
       "      <td>754.023511</td>\n",
       "      <td>19.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1128</th>\n",
       "      <td>fe751bcd59ea39c02069bbc276492daa502ef01d</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>709.134043</td>\n",
       "      <td>698.534043</td>\n",
       "      <td>698.534043</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1129</th>\n",
       "      <td>ff3e086b5016e9175c0025a60b8f6db90bb1ef6e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Mohave</td>\n",
       "      <td>692.604396</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>693.626374</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1130</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>689.658872</td>\n",
       "      <td>815.000000</td>\n",
       "      <td>691.040578</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1131</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>687.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7494 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00abe73a91e4a0012069878eba34b166551cce4d   feb  2017     auto   \n",
       "1     015b901f68620d06275e3b7aa91aaf7712578c24   feb  2017     auto   \n",
       "2     01e3ae470e88aac33b2483a97f19afafb926128c   feb  2017     auto   \n",
       "3     01e3ae470e88aac33b2483a97f19afafb926128c   feb  2017     auto   \n",
       "4     0222630f87433cb6a60e0fe83c9bc2233a842fce   feb  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1127  fe22e9c2412fbf136c42924645795c639968ca3a   feb  2021     heat   \n",
       "1128  fe751bcd59ea39c02069bbc276492daa502ef01d   feb  2021     heat   \n",
       "1129  ff3e086b5016e9175c0025a60b8f6db90bb1ef6e   feb  2021     auto   \n",
       "1130  ffec90658ad5a300da53b43a28fe991808fb4006   feb  2021     auto   \n",
       "1131  ffec90658ad5a300da53b43a28fe991808fb4006   feb  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold        Tucson            688.600000   \n",
       "1             hold          Mesa            698.472727   \n",
       "2             auto      Chandler            735.669477   \n",
       "3             hold      Chandler            735.939170   \n",
       "4             auto       Phoenix            738.236246   \n",
       "...            ...           ...                   ...   \n",
       "1127          hold      Chandler            753.611285   \n",
       "1128          hold       Gilbert            709.134043   \n",
       "1129          hold   Fort Mohave            692.604396   \n",
       "1130          hold  Sierra Vista            689.658872   \n",
       "1131          hold  Sierra Vista            687.000000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      770.000000                   680.000000   \n",
       "1                      721.672727                   658.400000   \n",
       "2                      778.387015                   727.600337   \n",
       "3                      778.408579                   727.329114   \n",
       "4                      757.640777                   753.097087   \n",
       "...                           ...                          ...   \n",
       "1127                   754.023511                   754.023511   \n",
       "1128                   698.534043                   698.534043   \n",
       "1129                   780.000000                   693.626374   \n",
       "1130                   815.000000                   691.040578   \n",
       "1131                   690.000000                   690.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False             True  \n",
       "1                    10.0             False         False            False  \n",
       "2                    15.0             False         False            False  \n",
       "3                    15.0             False         False            False  \n",
       "4                     0.0              True         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1127                 19.0              True         False            False  \n",
       "1128                  9.0             False         False            False  \n",
       "1129                 25.0             False         False            False  \n",
       "1130                  0.0             False         False            False  \n",
       "1131                  0.0             False         False            False  \n",
       "\n",
       "[7494 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/feb/\" + file)\n",
    "    AZ_feb = pd.concat([AZ_feb, df])\n",
    "    \n",
    "AZ_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_feb.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/AZ-day/2017-jun-day-AZ.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cb3d252b40e33c9b8240639fcbd3b27d9a23cc53</td>\n",
       "      <td>2017-06-18 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>759</td>\n",
       "      <td>729</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1186bfe43eb8c0463d3033d0ed62173252c60f1c</td>\n",
       "      <td>2017-06-02 11:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>782</td>\n",
       "      <td>785</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e147eeb021d885f460139447aa11e27eabe23273</td>\n",
       "      <td>2017-06-14 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>772</td>\n",
       "      <td>772</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>621250a303aedc23ae761ed6d885dc2656ca780d</td>\n",
       "      <td>2017-06-26 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>715</td>\n",
       "      <td>655</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Show Low</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0d70caae6cc9824cf120c4e92cf5e78fc7f933b4</td>\n",
       "      <td>2017-06-10 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>770</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Queen Creek</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559468</th>\n",
       "      <td>6d9610e892ce92158aaf7403a5ec4ee92bb3e53a</td>\n",
       "      <td>2017-06-30 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559469</th>\n",
       "      <td>01e3ae470e88aac33b2483a97f19afafb926128c</td>\n",
       "      <td>2017-06-24 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559470</th>\n",
       "      <td>a953a8b49c2eb9194208c2263b15f5b14b526393</td>\n",
       "      <td>2017-06-11 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559471</th>\n",
       "      <td>3f39761de407cc12effcc3c99a2bd59d820fd619</td>\n",
       "      <td>2017-06-29 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Glendale</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559472</th>\n",
       "      <td>5f37a90ab75318f95f477eceeea7997fbecfe523</td>\n",
       "      <td>2017-06-21 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>530829 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       cb3d252b40e33c9b8240639fcbd3b27d9a23cc53  2017-06-18 16:40:00 UTC   \n",
       "3       1186bfe43eb8c0463d3033d0ed62173252c60f1c  2017-06-02 11:55:00 UTC   \n",
       "4       e147eeb021d885f460139447aa11e27eabe23273  2017-06-14 14:20:00 UTC   \n",
       "5       621250a303aedc23ae761ed6d885dc2656ca780d  2017-06-26 14:50:00 UTC   \n",
       "6       0d70caae6cc9824cf120c4e92cf5e78fc7f933b4  2017-06-10 17:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "559468  6d9610e892ce92158aaf7403a5ec4ee92bb3e53a  2017-06-30 17:05:00 UTC   \n",
       "559469  01e3ae470e88aac33b2483a97f19afafb926128c  2017-06-24 10:40:00 UTC   \n",
       "559470  a953a8b49c2eb9194208c2263b15f5b14b526393  2017-06-11 17:10:00 UTC   \n",
       "559471  3f39761de407cc12effcc3c99a2bd59d820fd619  2017-06-29 19:00:00 UTC   \n",
       "559472  5f37a90ab75318f95f477eceeea7997fbecfe523  2017-06-21 14:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          cool          auto               762                      759   \n",
       "3          auto          hold               782                      785   \n",
       "4          cool          hold               768                      772   \n",
       "5          auto          auto               714                      715   \n",
       "6          cool          auto               770                      765   \n",
       "...         ...           ...               ...                      ...   \n",
       "559468     cool          hold               760                      760   \n",
       "559469     cool          auto               770                      770   \n",
       "559470     cool          hold               761                      760   \n",
       "559471     cool          hold               761                      760   \n",
       "559472     cool          auto               725                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "1                           729            AZ       Tucson   \n",
       "3                           715            AZ      Phoenix   \n",
       "4                           772            AZ         Mesa   \n",
       "5                           655            AZ     Show Low   \n",
       "6                           715            AZ  Queen Creek   \n",
       "...                         ...           ...          ...   \n",
       "559468                      760            AZ      Phoenix   \n",
       "559469                      760            AZ     Chandler   \n",
       "559470                      760            AZ       Peoria   \n",
       "559471                      760            AZ     Glendale   \n",
       "559472                      760            AZ        Tempe   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        10             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        40             False         False             True   \n",
       "5                        10             False         False            False   \n",
       "6                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "559468                   30             False         False             True   \n",
       "559469                   15             False         False            False   \n",
       "559470                   15             False         False            False   \n",
       "559471                   45             False         False             True   \n",
       "559472                   25             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "559468                  Electric  \n",
       "559469                       Gas  \n",
       "559470                       Gas  \n",
       "559471                  Electric  \n",
       "559472                  Electric  \n",
       "\n",
       "[530829 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/AZ/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/AZ-day/2018-jun-day-AZ.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>14936a3e0bf350ef87ccaddad0014c7a4b51da2a</td>\n",
       "      <td>2018-06-07 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>679</td>\n",
       "      <td>675</td>\n",
       "      <td>655</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14936a3e0bf350ef87ccaddad0014c7a4b51da2a</td>\n",
       "      <td>2018-06-24 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>675</td>\n",
       "      <td>655</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b67f4a3b9ed33a01822aa688dcd6816fbf59747c</td>\n",
       "      <td>2018-06-13 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>775</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>015b901f68620d06275e3b7aa91aaf7712578c24</td>\n",
       "      <td>2018-06-14 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>40309acd974cf0c593acdcc14f464c0721ca59a7</td>\n",
       "      <td>2018-06-12 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>765</td>\n",
       "      <td>655</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994831</th>\n",
       "      <td>1186bfe43eb8c0463d3033d0ed62173252c60f1c</td>\n",
       "      <td>2018-06-16 16:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>785</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994832</th>\n",
       "      <td>63be77a20fbfa6fbe35c3ce0abbc5bde444632c8</td>\n",
       "      <td>2018-06-11 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994833</th>\n",
       "      <td>5df58e41f64ee38403723e3c782e58a24e1644f3</td>\n",
       "      <td>2018-06-19 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Marana</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994834</th>\n",
       "      <td>85aa2e341897348088b381f0bd533354fe1262e8</td>\n",
       "      <td>2018-06-10 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>994835</th>\n",
       "      <td>98a2ae963be11439b5c112055d0eb7d5ece804ba</td>\n",
       "      <td>2018-06-15 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>938412 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       14936a3e0bf350ef87ccaddad0014c7a4b51da2a  2018-06-07 14:15:00 UTC   \n",
       "1       14936a3e0bf350ef87ccaddad0014c7a4b51da2a  2018-06-24 15:00:00 UTC   \n",
       "3       b67f4a3b9ed33a01822aa688dcd6816fbf59747c  2018-06-13 17:45:00 UTC   \n",
       "4       015b901f68620d06275e3b7aa91aaf7712578c24  2018-06-14 17:35:00 UTC   \n",
       "5       40309acd974cf0c593acdcc14f464c0721ca59a7  2018-06-12 14:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "994831  1186bfe43eb8c0463d3033d0ed62173252c60f1c  2018-06-16 16:55:00 UTC   \n",
       "994832  63be77a20fbfa6fbe35c3ce0abbc5bde444632c8  2018-06-11 19:40:00 UTC   \n",
       "994833  5df58e41f64ee38403723e3c782e58a24e1644f3  2018-06-19 17:45:00 UTC   \n",
       "994834  85aa2e341897348088b381f0bd533354fe1262e8  2018-06-10 15:00:00 UTC   \n",
       "994835  98a2ae963be11439b5c112055d0eb7d5ece804ba  2018-06-15 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               679                      675   \n",
       "1          cool          auto               677                      675   \n",
       "3          auto          hold               778                      775   \n",
       "4          auto          hold               735                      735   \n",
       "5          auto          hold               769                      765   \n",
       "...         ...           ...               ...                      ...   \n",
       "994831     cool          hold               785                      780   \n",
       "994832     cool          hold               762                      760   \n",
       "994833     cool          hold               764                      760   \n",
       "994834     cool          hold               766                      760   \n",
       "994835     cool          hold               759                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           655            AZ  Cave Creek   \n",
       "1                           655            AZ  Cave Creek   \n",
       "3                           705            AZ     Gilbert   \n",
       "4                           665            AZ        Mesa   \n",
       "5                           655            AZ    Chandler   \n",
       "...                         ...           ...         ...   \n",
       "994831                      760            AZ     Phoenix   \n",
       "994832                      760            AZ       Tempe   \n",
       "994833                      760            AZ      Marana   \n",
       "994834                      760            AZ      Peoria   \n",
       "994835                      760            AZ     Buckeye   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        27             False         False            False   \n",
       "1                        27             False         False            False   \n",
       "3                        30             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "5                         6             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "994831                    0             False         False            False   \n",
       "994832                   40             False         False            False   \n",
       "994833                   10             False         False            False   \n",
       "994834                   35              True         False             True   \n",
       "994835                   17             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "994831                       Gas  \n",
       "994832                       Gas  \n",
       "994833                       Gas  \n",
       "994834                  Electric  \n",
       "994835                       Gas  \n",
       "\n",
       "[938412 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/AZ/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/AZ-day/2019-jun-day-AZ.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a468a2af17424e6cf7a9e56d0221ffcf4c822c39</td>\n",
       "      <td>2019-06-24 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>786</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cc95eb27a8c93eac6357ca193c380315625a3850</td>\n",
       "      <td>2019-06-07 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>793</td>\n",
       "      <td>790</td>\n",
       "      <td>788</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6c24901ddb6616cdbca225d4593a43e8eedc9ab7</td>\n",
       "      <td>2019-06-12 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>794</td>\n",
       "      <td>846</td>\n",
       "      <td>842</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5b35014886264cf14e65d695dbe5b5d483031d73</td>\n",
       "      <td>2019-06-23 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>837</td>\n",
       "      <td>846</td>\n",
       "      <td>749</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2019-06-14 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>782</td>\n",
       "      <td>795</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1419202</th>\n",
       "      <td>5298e410369c7cf5a7a5e335d80b650977fa2289</td>\n",
       "      <td>2019-06-30 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1419203</th>\n",
       "      <td>015dc7c3497b4b091cd292a54c3fe1e48b73b2aa</td>\n",
       "      <td>2019-06-01 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1419204</th>\n",
       "      <td>f93883af48d15a1a6eb7d6d6afff070958c62d7a</td>\n",
       "      <td>2019-06-20 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>793</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1419205</th>\n",
       "      <td>1b941760cdf1a4e0d4b887a71a6fa78c86796db0</td>\n",
       "      <td>2019-06-15 12:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Oro Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1419206</th>\n",
       "      <td>471cbc673728ab126ef8748c407cd6fff5be48b7</td>\n",
       "      <td>2019-06-23 07:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1331301 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a468a2af17424e6cf7a9e56d0221ffcf4c822c39  2019-06-24 19:10:00 UTC   \n",
       "1        cc95eb27a8c93eac6357ca193c380315625a3850  2019-06-07 18:35:00 UTC   \n",
       "2        6c24901ddb6616cdbca225d4593a43e8eedc9ab7  2019-06-12 13:55:00 UTC   \n",
       "3        5b35014886264cf14e65d695dbe5b5d483031d73  2019-06-23 16:30:00 UTC   \n",
       "5        80467f30cce2c321e9e05405960ce92a377f38bc  2019-06-14 15:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1419202  5298e410369c7cf5a7a5e335d80b650977fa2289  2019-06-30 19:40:00 UTC   \n",
       "1419203  015dc7c3497b4b091cd292a54c3fe1e48b73b2aa  2019-06-01 19:15:00 UTC   \n",
       "1419204  f93883af48d15a1a6eb7d6d6afff070958c62d7a  2019-06-20 18:00:00 UTC   \n",
       "1419205  1b941760cdf1a4e0d4b887a71a6fa78c86796db0  2019-06-15 12:20:00 UTC   \n",
       "1419206  471cbc673728ab126ef8748c407cd6fff5be48b7  2019-06-23 07:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               786                      820   \n",
       "1           cool          hold               793                      790   \n",
       "2           cool          auto               794                      846   \n",
       "3           cool          hold               837                      846   \n",
       "5           auto          hold               782                      795   \n",
       "...          ...           ...               ...                      ...   \n",
       "1419202     cool          hold               766                      760   \n",
       "1419203     cool          hold               761                      760   \n",
       "1419204     cool          hold               793                      760   \n",
       "1419205     cool          hold               744                      760   \n",
       "1419206     cool          hold               758                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            820            AZ        Chandler   \n",
       "1                            788            AZ      Scottsdale   \n",
       "2                            842            AZ        Chandler   \n",
       "3                            749            AZ           Tempe   \n",
       "5                            685            AZ  San Tan Valley   \n",
       "...                          ...           ...             ...   \n",
       "1419202                      760            AZ        Chandler   \n",
       "1419203                      760            AZ         Phoenix   \n",
       "1419204                      760            AZ            Mesa   \n",
       "1419205                      760            AZ      Oro Valley   \n",
       "1419206                      760            AZ         Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         30              True         False             True   \n",
       "2                         10             False         False            False   \n",
       "3                          0             False         False            False   \n",
       "5                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1419202                    9              True         False             True   \n",
       "1419203                   30             False         False             True   \n",
       "1419204                   20             False         False            False   \n",
       "1419205                   10              True         False             True   \n",
       "1419206                   29              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1419202                  Electric  \n",
       "1419203                  Electric  \n",
       "1419204                       Gas  \n",
       "1419205                  Electric  \n",
       "1419206                       Gas  \n",
       "\n",
       "[1331301 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/AZ/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/AZ-day/2020-jun-day-AZ.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>425ac3626cf6e1ad590f194d5e1f78fba02b16cd</td>\n",
       "      <td>2020-06-23 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>788</td>\n",
       "      <td>785</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1f40eb23680da761bcf80f94387541f9241a9f72</td>\n",
       "      <td>2020-06-12 07:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b00f9da3121841492eef01ad8c9fbdece5e6b03e</td>\n",
       "      <td>2020-06-19 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>779</td>\n",
       "      <td>779</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0ac042f6fc949edceac5fb1ecda5a16bc4e45b77</td>\n",
       "      <td>2020-06-26 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>AZ</td>\n",
       "      <td>stanfield</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>53afc009cce56be1c6ee08f3038c4f3c3dd7ca73</td>\n",
       "      <td>2020-06-04 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>786</td>\n",
       "      <td>785</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Litchfield Park</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442780</th>\n",
       "      <td>9ebe523781d0d4ee256551ff0e0a60cb8eb15a4c</td>\n",
       "      <td>2020-06-16 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442781</th>\n",
       "      <td>bce12688151da76e5950316b49073323f3d98d94</td>\n",
       "      <td>2020-06-24 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442782</th>\n",
       "      <td>1186bfe43eb8c0463d3033d0ed62173252c60f1c</td>\n",
       "      <td>2020-06-01 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442783</th>\n",
       "      <td>f7c2b6873ba38921239c9aacc222817f0a9df1c2</td>\n",
       "      <td>2020-06-16 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1442784</th>\n",
       "      <td>ed41f989f7bee66128a93657cdf0206429c7a2dd</td>\n",
       "      <td>2020-06-04 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Marana</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1348117 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        425ac3626cf6e1ad590f194d5e1f78fba02b16cd  2020-06-23 15:45:00 UTC   \n",
       "1        1f40eb23680da761bcf80f94387541f9241a9f72  2020-06-12 07:35:00 UTC   \n",
       "2        b00f9da3121841492eef01ad8c9fbdece5e6b03e  2020-06-19 13:30:00 UTC   \n",
       "4        0ac042f6fc949edceac5fb1ecda5a16bc4e45b77  2020-06-26 18:25:00 UTC   \n",
       "5        53afc009cce56be1c6ee08f3038c4f3c3dd7ca73  2020-06-04 15:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1442780  9ebe523781d0d4ee256551ff0e0a60cb8eb15a4c  2020-06-16 14:55:00 UTC   \n",
       "1442781  bce12688151da76e5950316b49073323f3d98d94  2020-06-24 16:35:00 UTC   \n",
       "1442782  1186bfe43eb8c0463d3033d0ed62173252c60f1c  2020-06-01 14:45:00 UTC   \n",
       "1442783  f7c2b6873ba38921239c9aacc222817f0a9df1c2  2020-06-16 17:25:00 UTC   \n",
       "1442784  ed41f989f7bee66128a93657cdf0206429c7a2dd  2020-06-04 13:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               788                      785   \n",
       "1           cool          hold               762                      765   \n",
       "2           cool          hold               776                      779   \n",
       "4           cool          hold               758                      755   \n",
       "5           auto          hold               786                      785   \n",
       "...          ...           ...               ...                      ...   \n",
       "1442780     cool          auto               763                      760   \n",
       "1442781     cool          auto               755                      760   \n",
       "1442782     cool          hold               775                      770   \n",
       "1442783     cool          hold               749                      760   \n",
       "1442784     cool          hold               764                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            705            AZ         Chandler   \n",
       "1                            765            AZ       Scottsdale   \n",
       "2                            779            AZ            Tempe   \n",
       "4                            755            AZ        stanfield   \n",
       "5                            705            AZ  Litchfield Park   \n",
       "...                          ...           ...              ...   \n",
       "1442780                      760            AZ          Phoenix   \n",
       "1442781                      760            AZ           Tucson   \n",
       "1442782                      760            AZ          Phoenix   \n",
       "1442783                      760            AZ          Phoenix   \n",
       "1442784                      760            AZ           Marana   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10              True         False            False   \n",
       "1                          0             False         False             True   \n",
       "2                          0             False         False             True   \n",
       "4                         10              True         False             True   \n",
       "5                         15             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1442780                    0              True         False             True   \n",
       "1442781                   45             False         False            False   \n",
       "1442782                    0             False         False            False   \n",
       "1442783                   20             False         False            False   \n",
       "1442784                   10             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "4                        Electric  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1442780                  Electric  \n",
       "1442781                       Gas  \n",
       "1442782                       Gas  \n",
       "1442783                       Gas  \n",
       "1442784                       Gas  \n",
       "\n",
       "[1348117 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/AZ/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/AZ-day/2021-jun-day-AZ.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f5fd44aaddb7f2d604ef7ddc97e23187531d6be1</td>\n",
       "      <td>2021-06-10 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4fcefbf541c1e66392577df6912269611736c5b0</td>\n",
       "      <td>2021-06-14 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>826</td>\n",
       "      <td>830</td>\n",
       "      <td>800</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0951b1fd22270363781e81814be00d51bc77124d</td>\n",
       "      <td>2021-06-25 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>735</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1a39154f7267bea2a58b125d47afb78b6db01d6a</td>\n",
       "      <td>2021-06-16 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>790</td>\n",
       "      <td>734</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Kingman</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1a672b2d2d457ef5a28cf32df6f993b9e8f9095b</td>\n",
       "      <td>2021-06-14 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>753</td>\n",
       "      <td>753</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Bisbee</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920526</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2021-06-20 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920527</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2021-06-26 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920528</th>\n",
       "      <td>e9726b9c4d69508d33a77d75c05e84f0e2bba5d8</td>\n",
       "      <td>2021-06-01 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920529</th>\n",
       "      <td>e9726b9c4d69508d33a77d75c05e84f0e2bba5d8</td>\n",
       "      <td>2021-06-15 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920530</th>\n",
       "      <td>45da57e128203e5f592a15106401255eb70d565d</td>\n",
       "      <td>2021-06-25 07:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>866480 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f5fd44aaddb7f2d604ef7ddc97e23187531d6be1  2021-06-10 19:50:00 UTC   \n",
       "1       4fcefbf541c1e66392577df6912269611736c5b0  2021-06-14 12:25:00 UTC   \n",
       "2       0951b1fd22270363781e81814be00d51bc77124d  2021-06-25 16:50:00 UTC   \n",
       "3       1a39154f7267bea2a58b125d47afb78b6db01d6a  2021-06-16 13:40:00 UTC   \n",
       "4       1a672b2d2d457ef5a28cf32df6f993b9e8f9095b  2021-06-14 14:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "920526  80467f30cce2c321e9e05405960ce92a377f38bc  2021-06-20 18:10:00 UTC   \n",
       "920527  80467f30cce2c321e9e05405960ce92a377f38bc  2021-06-26 16:20:00 UTC   \n",
       "920528  e9726b9c4d69508d33a77d75c05e84f0e2bba5d8  2021-06-01 18:55:00 UTC   \n",
       "920529  e9726b9c4d69508d33a77d75c05e84f0e2bba5d8  2021-06-15 15:05:00 UTC   \n",
       "920530  45da57e128203e5f592a15106401255eb70d565d  2021-06-25 07:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               761                      757   \n",
       "1          cool          hold               826                      830   \n",
       "2          auto          hold               737                      735   \n",
       "3          cool          hold               760                      790   \n",
       "4          cool          hold               757                      753   \n",
       "...         ...           ...               ...                      ...   \n",
       "920526     cool          hold               768                      765   \n",
       "920527     cool          hold               761                      765   \n",
       "920528     cool          hold               765                      765   \n",
       "920529     cool          hold               761                      765   \n",
       "920530     cool          hold               764                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           757            AZ            Yuma   \n",
       "1                           800            AZ         Gilbert   \n",
       "2                           665            AZ          Tucson   \n",
       "3                           734            AZ         Kingman   \n",
       "4                           753            AZ          Bisbee   \n",
       "...                         ...           ...             ...   \n",
       "920526                      765            AZ  San Tan Valley   \n",
       "920527                      765            AZ  San Tan Valley   \n",
       "920528                      765            AZ         Phoenix   \n",
       "920529                      765            AZ         Phoenix   \n",
       "920530                      765            AZ         Buckeye   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False             True   \n",
       "1                        15             False         False             True   \n",
       "2                        29             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "920526                   10             False         False            False   \n",
       "920527                   10             False         False            False   \n",
       "920528                    0             False         False             True   \n",
       "920529                    0             False         False             True   \n",
       "920530                    7             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "920526                       Gas  \n",
       "920527                       Gas  \n",
       "920528                  Electric  \n",
       "920529                  Electric  \n",
       "920530                       Gas  \n",
       "\n",
       "[866480 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/AZ/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>780.433383</td>\n",
       "      <td>784.720808</td>\n",
       "      <td>757.682635</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>781.662791</td>\n",
       "      <td>782.790698</td>\n",
       "      <td>780.930233</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>765.594444</td>\n",
       "      <td>764.182222</td>\n",
       "      <td>687.833333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01016c682a7be3fa75027c0ebd9edf413cf2459e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>761.009804</td>\n",
       "      <td>760.294118</td>\n",
       "      <td>689.754902</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01016c682a7be3fa75027c0ebd9edf413cf2459e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>766.282132</td>\n",
       "      <td>767.222571</td>\n",
       "      <td>765.783699</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1253</th>\n",
       "      <td>fe7d60cb7fe777a80cd29f1df2350728a8ac3381</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>747.600000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1254</th>\n",
       "      <td>ff2a7e846baca4fde6914b30a69e4f6327d76db5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>784.859122</td>\n",
       "      <td>788.730947</td>\n",
       "      <td>783.519630</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1255</th>\n",
       "      <td>ffc69aa005f871624e7187d94b86f1d795252fd7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>794.774709</td>\n",
       "      <td>793.793605</td>\n",
       "      <td>786.513081</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1256</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>756.847215</td>\n",
       "      <td>757.487925</td>\n",
       "      <td>685.766387</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1257</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>748.553942</td>\n",
       "      <td>747.784232</td>\n",
       "      <td>747.522822</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8798 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00ab105be10ba80cca71a5e8d83088116fe17003   jun  2017     cool   \n",
       "1     00ab105be10ba80cca71a5e8d83088116fe17003   jun  2017     cool   \n",
       "2     00abe73a91e4a0012069878eba34b166551cce4d   jun  2017     auto   \n",
       "3     01016c682a7be3fa75027c0ebd9edf413cf2459e   jun  2017     cool   \n",
       "4     01016c682a7be3fa75027c0ebd9edf413cf2459e   jun  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1253  fe7d60cb7fe777a80cd29f1df2350728a8ac3381   jun  2021     cool   \n",
       "1254  ff2a7e846baca4fde6914b30a69e4f6327d76db5   jun  2021     cool   \n",
       "1255  ffc69aa005f871624e7187d94b86f1d795252fd7   jun  2021     cool   \n",
       "1256  ffec90658ad5a300da53b43a28fe991808fb4006   jun  2021     auto   \n",
       "1257  fffd2d78fc884b609293e46bfc90641926869ebb   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto        Tucson            780.433383   \n",
       "1             hold        Tucson            781.662791   \n",
       "2             hold        Tucson            765.594444   \n",
       "3             auto        Peoria            761.009804   \n",
       "4             hold        Peoria            766.282132   \n",
       "...            ...           ...                   ...   \n",
       "1253          hold      Surprise            747.600000   \n",
       "1254          hold       Phoenix            784.859122   \n",
       "1255          hold       Phoenix            794.774709   \n",
       "1256          hold  Sierra Vista            756.847215   \n",
       "1257          hold    Scottsdale            748.553942   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      784.720808                   757.682635   \n",
       "1                      782.790698                   780.930233   \n",
       "2                      764.182222                   687.833333   \n",
       "3                      760.294118                   689.754902   \n",
       "4                      767.222571                   765.783699   \n",
       "...                           ...                          ...   \n",
       "1253                   750.000000                   750.000000   \n",
       "1254                   788.730947                   783.519630   \n",
       "1255                   793.793605                   786.513081   \n",
       "1256                   757.487925                   685.766387   \n",
       "1257                   747.784232                   747.522822   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    10.0             False         False            False  \n",
       "1                    10.0             False         False            False  \n",
       "2                     0.0             False         False             True  \n",
       "3                     0.0             False         False            False  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1253                  0.0             False         False            False  \n",
       "1254                  0.0             False         False            False  \n",
       "1255                 50.0              True         False             True  \n",
       "1256                  0.0             False         False            False  \n",
       "1257                 30.0             False         False            False  \n",
       "\n",
       "[8798 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/jun/\" + file)\n",
    "    AZ_jun = pd.concat([AZ_jun, df])\n",
    "    \n",
    "AZ_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_jun.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/AZ-day/2017-jul-day-AZ.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f519906973abc35eea9e76bc0fa37def65df8778</td>\n",
       "      <td>2017-07-24 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chino Valley</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31743671a90ceecea076c02a12b72afd28e03fef</td>\n",
       "      <td>2017-07-24 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sun City West</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1e050c07022faad3babc87a30a28538466629590</td>\n",
       "      <td>2017-07-20 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Litchfield park</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5d27ef4d268c5bcee4469a6b2ececa65a6f5adf3</td>\n",
       "      <td>2017-07-03 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>749</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Desert Hills</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4f0fefd7929a9b1a58368fd50b62e01c0ca7fc8a</td>\n",
       "      <td>2017-07-20 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>830</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sun City</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637629</th>\n",
       "      <td>6dcb7638afa9e750b3f74113a3ebcc2b58772f1f</td>\n",
       "      <td>2017-07-02 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637630</th>\n",
       "      <td>947147f97498df3ec675a6375b3efc0d7472b9f5</td>\n",
       "      <td>2017-07-25 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>790</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637631</th>\n",
       "      <td>7a1f42fafd9967610184b75ce8c477548c3dd980</td>\n",
       "      <td>2017-07-13 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>784</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637632</th>\n",
       "      <td>7a1f42fafd9967610184b75ce8c477548c3dd980</td>\n",
       "      <td>2017-07-27 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>784</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637633</th>\n",
       "      <td>7f8fc16db219e6bd7bba790ade2a5d3e97ea0433</td>\n",
       "      <td>2017-07-04 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>749</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>601780 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       f519906973abc35eea9e76bc0fa37def65df8778  2017-07-24 16:15:00 UTC   \n",
       "2       31743671a90ceecea076c02a12b72afd28e03fef  2017-07-24 18:15:00 UTC   \n",
       "4       1e050c07022faad3babc87a30a28538466629590  2017-07-20 19:45:00 UTC   \n",
       "5       5d27ef4d268c5bcee4469a6b2ececa65a6f5adf3  2017-07-03 19:05:00 UTC   \n",
       "6       4f0fefd7929a9b1a58368fd50b62e01c0ca7fc8a  2017-07-20 17:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "637629  6dcb7638afa9e750b3f74113a3ebcc2b58772f1f  2017-07-02 19:10:00 UTC   \n",
       "637630  947147f97498df3ec675a6375b3efc0d7472b9f5  2017-07-25 14:10:00 UTC   \n",
       "637631  7a1f42fafd9967610184b75ce8c477548c3dd980  2017-07-13 11:40:00 UTC   \n",
       "637632  7a1f42fafd9967610184b75ce8c477548c3dd980  2017-07-27 11:40:00 UTC   \n",
       "637633  7f8fc16db219e6bd7bba790ade2a5d3e97ea0433  2017-07-04 13:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          hold               699                      700   \n",
       "2          cool          hold               741                      760   \n",
       "4          cool          hold               775                      770   \n",
       "5          cool          auto               749                      750   \n",
       "6          cool          auto               783                      830   \n",
       "...         ...           ...               ...                      ...   \n",
       "637629     cool          hold               719                      720   \n",
       "637630     cool          auto               753                      790   \n",
       "637631     cool          hold               784                      780   \n",
       "637632     cool          hold               784                      780   \n",
       "637633     cool          auto               749                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState             City  \\\n",
       "1                           640            AZ     Chino Valley   \n",
       "2                           760            AZ    Sun City West   \n",
       "4                           770            AZ  Litchfield park   \n",
       "5                           680            AZ     Desert Hills   \n",
       "6                           640            AZ         Sun City   \n",
       "...                         ...           ...              ...   \n",
       "637629                      720            AZ   San Tan Valley   \n",
       "637630                      640            AZ   San Tan Valley   \n",
       "637631                      780            AZ   San Tan Valley   \n",
       "637632                      780            AZ   San Tan Valley   \n",
       "637633                      680            AZ   San Tan Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        15             False         False            False   \n",
       "2                        15             False         False            False   \n",
       "4                        15             False         False             True   \n",
       "5                        25             False         False             True   \n",
       "6                        47             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "637629                    5             False         False            False   \n",
       "637630                    0             False         False            False   \n",
       "637631                   10              True         False            False   \n",
       "637632                   10              True         False            False   \n",
       "637633                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "4                       Electric  \n",
       "5                       Electric  \n",
       "6                       Electric  \n",
       "...                          ...  \n",
       "637629                       Gas  \n",
       "637630                       Gas  \n",
       "637631                       Gas  \n",
       "637632                       Gas  \n",
       "637633                       Gas  \n",
       "\n",
       "[601780 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/AZ/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/AZ-day/2018-jul-day-AZ.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2bafaa9268961800b3769c287ee288710b0dd7b7</td>\n",
       "      <td>2018-07-24 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>800</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c98c6aef64866d625db4925f4e464eaa2ad62721</td>\n",
       "      <td>2018-07-14 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>768</td>\n",
       "      <td>738</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f140a632c95e3358d828843b8b5559275d622f2c</td>\n",
       "      <td>2018-07-03 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54c8534059657d531f8491829a02bccabc7e002f</td>\n",
       "      <td>2018-07-07 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>790</td>\n",
       "      <td>795</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>aa969b7f663b62c6a66c7ad2d00fe27a1f2e92ba</td>\n",
       "      <td>2018-07-05 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125521</th>\n",
       "      <td>a8691bd122326b599780901123d812b26287c1f1</td>\n",
       "      <td>2018-07-10 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Lake Havasu City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125522</th>\n",
       "      <td>f0b99206fd4d1382147b8d70a1ef35de1972c60d</td>\n",
       "      <td>2018-07-19 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125523</th>\n",
       "      <td>9db8fee199126f595835859da82784f93742e8d1</td>\n",
       "      <td>2018-07-21 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125524</th>\n",
       "      <td>f439ecc40f055d4bcf4fcfd020f9ac76d7c99506</td>\n",
       "      <td>2018-07-21 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1125525</th>\n",
       "      <td>cbb358f6d3fbf33a0b6251454b2de8fa52b08c9b</td>\n",
       "      <td>2018-07-17 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1046993 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        2bafaa9268961800b3769c287ee288710b0dd7b7  2018-07-24 15:45:00 UTC   \n",
       "1        c98c6aef64866d625db4925f4e464eaa2ad62721  2018-07-14 17:45:00 UTC   \n",
       "2        f140a632c95e3358d828843b8b5559275d622f2c  2018-07-03 14:25:00 UTC   \n",
       "3        54c8534059657d531f8491829a02bccabc7e002f  2018-07-07 16:50:00 UTC   \n",
       "4        aa969b7f663b62c6a66c7ad2d00fe27a1f2e92ba  2018-07-05 15:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1125521  a8691bd122326b599780901123d812b26287c1f1  2018-07-10 18:00:00 UTC   \n",
       "1125522  f0b99206fd4d1382147b8d70a1ef35de1972c60d  2018-07-19 15:25:00 UTC   \n",
       "1125523  9db8fee199126f595835859da82784f93742e8d1  2018-07-21 16:45:00 UTC   \n",
       "1125524  f439ecc40f055d4bcf4fcfd020f9ac76d7c99506  2018-07-21 14:55:00 UTC   \n",
       "1125525  cbb358f6d3fbf33a0b6251454b2de8fa52b08c9b  2018-07-17 14:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               800                      685   \n",
       "1           cool          hold               752                      768   \n",
       "2           cool          auto               820                      820   \n",
       "3           auto          hold               790                      795   \n",
       "4           cool          hold               702                      695   \n",
       "...          ...           ...               ...                      ...   \n",
       "1125521     cool          auto               764                      760   \n",
       "1125522     cool          hold               757                      760   \n",
       "1125523     cool          hold               760                      760   \n",
       "1125524     cool          hold               730                      760   \n",
       "1125525     cool          auto               756                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            685            AZ           Gilbert   \n",
       "1                            738            AZ           Phoenix   \n",
       "2                            820            AZ          Chandler   \n",
       "3                            705            AZ           Gilbert   \n",
       "4                            695            AZ        Scottsdale   \n",
       "...                          ...           ...               ...   \n",
       "1125521                      760            AZ  Lake Havasu City   \n",
       "1125522                      760            AZ         Sahuarita   \n",
       "1125523                      760            AZ             Tempe   \n",
       "1125524                      760            AZ           Gilbert   \n",
       "1125525                      760            AZ            Tucson   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30             False         False            False   \n",
       "1                         20              True         False             True   \n",
       "2                          0              True         False            False   \n",
       "3                          6             False         False            False   \n",
       "4                         35             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1125521                   10             False         False             True   \n",
       "1125522                   25             False         False            False   \n",
       "1125523                    0             False         False            False   \n",
       "1125524                   20             False         False            False   \n",
       "1125525                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1125521                  Electric  \n",
       "1125522                       Gas  \n",
       "1125523                       Gas  \n",
       "1125524                       Gas  \n",
       "1125525                       Gas  \n",
       "\n",
       "[1046993 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/AZ/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/AZ-day/2019-jul-day-AZ.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e8ab3f9281e4b47e63004908f512772635f53361</td>\n",
       "      <td>2019-07-16 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>800</td>\n",
       "      <td>618</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Flagstaff</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bb1191aa00ff520c09a63cecc92aba1bf44072d1</td>\n",
       "      <td>2019-07-17 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>766</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d6f207d822f8b93da8a7257b5a3c6d0a6b652da4</td>\n",
       "      <td>2019-07-14 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>605</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Prescott Valley</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fd567104c456f96468cde6628db6d0dca4c940c9</td>\n",
       "      <td>2019-07-04 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>750</td>\n",
       "      <td>610</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c5f9575896ba4d402c5ad4a30d79fce757499378</td>\n",
       "      <td>2019-07-13 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>789</td>\n",
       "      <td>800</td>\n",
       "      <td>768</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508565</th>\n",
       "      <td>1a39154f7267bea2a58b125d47afb78b6db01d6a</td>\n",
       "      <td>2019-07-16 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>822</td>\n",
       "      <td>830</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Kingman</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508566</th>\n",
       "      <td>1b941760cdf1a4e0d4b887a71a6fa78c86796db0</td>\n",
       "      <td>2019-07-18 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Oro Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508567</th>\n",
       "      <td>efebef567ff3dd92fbc5816fa35ec010c08de832</td>\n",
       "      <td>2019-07-24 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508568</th>\n",
       "      <td>341ded0d08168fcb6d28b91d7640f936dfa91d38</td>\n",
       "      <td>2019-07-11 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>glendale</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1508569</th>\n",
       "      <td>d00a366d0be3bfeab230b62ff9a12f5338045ebd</td>\n",
       "      <td>2019-07-21 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1406545 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        e8ab3f9281e4b47e63004908f512772635f53361  2019-07-16 14:20:00 UTC   \n",
       "1        bb1191aa00ff520c09a63cecc92aba1bf44072d1  2019-07-17 15:30:00 UTC   \n",
       "2        d6f207d822f8b93da8a7257b5a3c6d0a6b652da4  2019-07-14 16:05:00 UTC   \n",
       "3        fd567104c456f96468cde6628db6d0dca4c940c9  2019-07-04 16:35:00 UTC   \n",
       "4        c5f9575896ba4d402c5ad4a30d79fce757499378  2019-07-13 17:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1508565  1a39154f7267bea2a58b125d47afb78b6db01d6a  2019-07-16 17:15:00 UTC   \n",
       "1508566  1b941760cdf1a4e0d4b887a71a6fa78c86796db0  2019-07-18 16:50:00 UTC   \n",
       "1508567  efebef567ff3dd92fbc5816fa35ec010c08de832  2019-07-24 16:25:00 UTC   \n",
       "1508568  341ded0d08168fcb6d28b91d7640f936dfa91d38  2019-07-11 14:35:00 UTC   \n",
       "1508569  d00a366d0be3bfeab230b62ff9a12f5338045ebd  2019-07-21 13:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               760                      800   \n",
       "1           cool          hold               774                      770   \n",
       "2           auto          hold               761                      760   \n",
       "3           auto          hold               763                      750   \n",
       "4           cool          auto               789                      800   \n",
       "...          ...           ...               ...                      ...   \n",
       "1508565     cool          auto               822                      830   \n",
       "1508566     cool          auto               764                      760   \n",
       "1508567     cool          hold               755                      760   \n",
       "1508568     cool          hold               764                      760   \n",
       "1508569     cool          auto               760                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            618            AZ        Flagstaff   \n",
       "1                            766            AZ          Gilbert   \n",
       "2                            605            AZ  Prescott Valley   \n",
       "3                            610            AZ        Sahuarita   \n",
       "4                            768            AZ          Phoenix   \n",
       "...                          ...           ...              ...   \n",
       "1508565                      760            AZ          Kingman   \n",
       "1508566                      760            AZ       Oro Valley   \n",
       "1508567                      760            AZ          Phoenix   \n",
       "1508568                      760            AZ        glendale    \n",
       "1508569                      760            AZ          Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5             False         False            False   \n",
       "1                          5             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         15             False         False            False   \n",
       "4                         30             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1508565                    0             False         False            False   \n",
       "1508566                   10              True         False             True   \n",
       "1508567                   10             False         False            False   \n",
       "1508568                   20             False         False            False   \n",
       "1508569                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1508565                       Gas  \n",
       "1508566                  Electric  \n",
       "1508567                       Gas  \n",
       "1508568                       Gas  \n",
       "1508569                       Gas  \n",
       "\n",
       "[1406545 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/AZ/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/AZ-day/2020-jul-day-AZ.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1697e33768e33a3eba2cdd4464849a40f6f3ebcc</td>\n",
       "      <td>2020-07-10 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>781</td>\n",
       "      <td>781</td>\n",
       "      <td>781</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Avondale</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0ea7bcc48ce9a7dfc94e145d807f7d4cbc7e687e</td>\n",
       "      <td>2020-07-18 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>765</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sedona</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4488951eac06434a289e3b0133ecbb01c17113f8</td>\n",
       "      <td>2020-07-01 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>63c57939a5b432f80fadb8501dcbfaab9c1ed4e2</td>\n",
       "      <td>2020-07-17 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>700</td>\n",
       "      <td>614</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4b56fe9b3c45ba417c432211405e5cfccbbeb452</td>\n",
       "      <td>2020-07-12 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>762</td>\n",
       "      <td>672</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504954</th>\n",
       "      <td>cbff3b7a1a08ff17cfc7b97d6b3eaa0461148f51</td>\n",
       "      <td>2020-07-31 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504955</th>\n",
       "      <td>c6b5a22e0e26e02336b5770e78a2b82597c0cee2</td>\n",
       "      <td>2020-07-10 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>772</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504956</th>\n",
       "      <td>028dd9fd947eb85d85eafe758967d10e3f3f39e8</td>\n",
       "      <td>2020-07-25 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504957</th>\n",
       "      <td>09f3dca8198990b9d2a91a2a71148338b3fa6a83</td>\n",
       "      <td>2020-07-16 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Queen Creek</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1504958</th>\n",
       "      <td>a0e3e68e778ef4da16bc4c6f87fe7a8af595b3c5</td>\n",
       "      <td>2020-07-23 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Yuma</td>\n",
       "      <td>70</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1413176 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        1697e33768e33a3eba2cdd4464849a40f6f3ebcc  2020-07-10 19:30:00 UTC   \n",
       "1        0ea7bcc48ce9a7dfc94e145d807f7d4cbc7e687e  2020-07-18 18:10:00 UTC   \n",
       "2        4488951eac06434a289e3b0133ecbb01c17113f8  2020-07-01 14:40:00 UTC   \n",
       "3        63c57939a5b432f80fadb8501dcbfaab9c1ed4e2  2020-07-17 13:45:00 UTC   \n",
       "4        4b56fe9b3c45ba417c432211405e5cfccbbeb452  2020-07-12 17:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1504954  cbff3b7a1a08ff17cfc7b97d6b3eaa0461148f51  2020-07-31 15:35:00 UTC   \n",
       "1504955  c6b5a22e0e26e02336b5770e78a2b82597c0cee2  2020-07-10 17:55:00 UTC   \n",
       "1504956  028dd9fd947eb85d85eafe758967d10e3f3f39e8  2020-07-25 19:10:00 UTC   \n",
       "1504957  09f3dca8198990b9d2a91a2a71148338b3fa6a83  2020-07-16 16:50:00 UTC   \n",
       "1504958  a0e3e68e778ef4da16bc4c6f87fe7a8af595b3c5  2020-07-23 16:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               781                      781   \n",
       "1           auto          hold               763                      765   \n",
       "2           auto          hold               746                      735   \n",
       "3           auto          hold               688                      700   \n",
       "4           cool          auto               758                      762   \n",
       "...          ...           ...               ...                      ...   \n",
       "1504954     cool          hold               761                      760   \n",
       "1504955     cool          auto               772                      760   \n",
       "1504956     cool          hold               775                      760   \n",
       "1504957     cool          hold               762                      760   \n",
       "1504958     cool          hold               775                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            781            AZ     Avondale   \n",
       "1                            685            AZ       Sedona   \n",
       "2                            685            AZ      Buckeye   \n",
       "3                            614            AZ    Sahuarita   \n",
       "4                            672            AZ     Surprise   \n",
       "...                          ...           ...          ...   \n",
       "1504954                      760            AZ   Scottsdale   \n",
       "1504955                      760            AZ     Chandler   \n",
       "1504956                      760            AZ        Tempe   \n",
       "1504957                      760            AZ  Queen Creek   \n",
       "1504958                      760            AZ         Yuma   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False             True   \n",
       "1                         29              True         False            False   \n",
       "2                          0              True         False             True   \n",
       "3                         15              True         False            False   \n",
       "4                         10             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1504954                    0             False         False            False   \n",
       "1504955                   10             False         False             True   \n",
       "1504956                   30             False         False             True   \n",
       "1504957                   10             False         False            False   \n",
       "1504958                   70              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1504954                       Gas  \n",
       "1504955                  Electric  \n",
       "1504956                  Electric  \n",
       "1504957                       Gas  \n",
       "1504958                  Electric  \n",
       "\n",
       "[1413176 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/AZ/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/AZ-day/2021-jul-day-AZ.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0004d19902138a2152dbbaeb5f71d652551f23b7</td>\n",
       "      <td>2021-07-20 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>745</td>\n",
       "      <td>645</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2021-07-01 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>85677b42fe74484c21b9fe7c7d8188aad018e304</td>\n",
       "      <td>2021-07-23 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>786</td>\n",
       "      <td>780</td>\n",
       "      <td>649</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4ad398cb1b6109ab5b5ffa77ef59959a25d4308f</td>\n",
       "      <td>2021-07-02 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>799</td>\n",
       "      <td>800</td>\n",
       "      <td>712</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3f36cfc86c1f4d9e3aeec7d993e47bd935eee9e0</td>\n",
       "      <td>2021-07-21 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>730</td>\n",
       "      <td>610</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881628</th>\n",
       "      <td>98a2ae963be11439b5c112055d0eb7d5ece804ba</td>\n",
       "      <td>2021-07-12 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881629</th>\n",
       "      <td>f1c616a72d276a71c989687091e2524140078322</td>\n",
       "      <td>2021-07-27 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881630</th>\n",
       "      <td>e8a0700ccb39c5956b4ffeea3e440bd34a9d2dc7</td>\n",
       "      <td>2021-07-20 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881631</th>\n",
       "      <td>8563313be1404e512485f7a4419744190cd7fc83</td>\n",
       "      <td>2021-07-20 09:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>881632</th>\n",
       "      <td>21d77c12474755ea59246f01b7431ef5f05fb46c</td>\n",
       "      <td>2021-07-11 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Anthem</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>837698 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0004d19902138a2152dbbaeb5f71d652551f23b7  2021-07-20 14:20:00 UTC   \n",
       "1       80467f30cce2c321e9e05405960ce92a377f38bc  2021-07-01 18:55:00 UTC   \n",
       "2       85677b42fe74484c21b9fe7c7d8188aad018e304  2021-07-23 14:10:00 UTC   \n",
       "3       4ad398cb1b6109ab5b5ffa77ef59959a25d4308f  2021-07-02 19:25:00 UTC   \n",
       "4       3f36cfc86c1f4d9e3aeec7d993e47bd935eee9e0  2021-07-21 16:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "881628  98a2ae963be11439b5c112055d0eb7d5ece804ba  2021-07-12 18:20:00 UTC   \n",
       "881629  f1c616a72d276a71c989687091e2524140078322  2021-07-27 14:55:00 UTC   \n",
       "881630  e8a0700ccb39c5956b4ffeea3e440bd34a9d2dc7  2021-07-20 13:50:00 UTC   \n",
       "881631  8563313be1404e512485f7a4419744190cd7fc83  2021-07-20 09:00:00 UTC   \n",
       "881632  21d77c12474755ea59246f01b7431ef5f05fb46c  2021-07-11 16:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               742                      745   \n",
       "1          cool          hold               766                      765   \n",
       "2          auto          hold               786                      780   \n",
       "3          cool          hold               799                      800   \n",
       "4          auto          hold               737                      730   \n",
       "...         ...           ...               ...                      ...   \n",
       "881628     cool          hold               752                      760   \n",
       "881629     cool          hold               752                      760   \n",
       "881630     cool          hold               763                      760   \n",
       "881631     cool          hold               754                      760   \n",
       "881632     cool          hold               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           645            AZ          Tucson   \n",
       "1                           765            AZ  San Tan Valley   \n",
       "2                           649            AZ            Mesa   \n",
       "3                           712            AZ        Surprise   \n",
       "4                           610            AZ         Phoenix   \n",
       "...                         ...           ...             ...   \n",
       "881628                      760            AZ         Buckeye   \n",
       "881629                      760            AZ            Mesa   \n",
       "881630                      760            AZ         Buckeye   \n",
       "881631                      760            AZ        Chandler   \n",
       "881632                      760            AZ          Anthem   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        39              True         False            False   \n",
       "3                        15             False         False            False   \n",
       "4                        39              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "881628                   17             False         False            False   \n",
       "881629                    0              True         False             True   \n",
       "881630                    5             False         False             True   \n",
       "881631                   45              True         False             True   \n",
       "881632                   20             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "881628                       Gas  \n",
       "881629                  Electric  \n",
       "881630                  Electric  \n",
       "881631                  Electric  \n",
       "881632                       Gas  \n",
       "\n",
       "[837698 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/AZ/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>789.124172</td>\n",
       "      <td>795.200331</td>\n",
       "      <td>780.149007</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>777.244898</td>\n",
       "      <td>777.530612</td>\n",
       "      <td>777.346939</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>762.216667</td>\n",
       "      <td>756.666667</td>\n",
       "      <td>683.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>765.257048</td>\n",
       "      <td>762.363184</td>\n",
       "      <td>676.230514</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01016c682a7be3fa75027c0ebd9edf413cf2459e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>767.103746</td>\n",
       "      <td>767.233429</td>\n",
       "      <td>766.662824</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1221</th>\n",
       "      <td>fe751bcd59ea39c02069bbc276492daa502ef01d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>772.022325</td>\n",
       "      <td>772.220725</td>\n",
       "      <td>709.993682</td>\n",
       "      <td>9.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1222</th>\n",
       "      <td>ff2a7e846baca4fde6914b30a69e4f6327d76db5</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>794.257252</td>\n",
       "      <td>798.941221</td>\n",
       "      <td>786.237405</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1223</th>\n",
       "      <td>ffc69aa005f871624e7187d94b86f1d795252fd7</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>797.165756</td>\n",
       "      <td>794.816029</td>\n",
       "      <td>783.870674</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1224</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>752.119628</td>\n",
       "      <td>752.822300</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1225</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>750.899824</td>\n",
       "      <td>749.143234</td>\n",
       "      <td>749.118629</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9181 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00ab105be10ba80cca71a5e8d83088116fe17003   jul  2017     cool   \n",
       "1     00ab105be10ba80cca71a5e8d83088116fe17003   jul  2017     cool   \n",
       "2     00abe73a91e4a0012069878eba34b166551cce4d   jul  2017     auto   \n",
       "3     00abe73a91e4a0012069878eba34b166551cce4d   jul  2017     auto   \n",
       "4     01016c682a7be3fa75027c0ebd9edf413cf2459e   jul  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1221  fe751bcd59ea39c02069bbc276492daa502ef01d   jul  2021     cool   \n",
       "1222  ff2a7e846baca4fde6914b30a69e4f6327d76db5   jul  2021     cool   \n",
       "1223  ffc69aa005f871624e7187d94b86f1d795252fd7   jul  2021     cool   \n",
       "1224  ffec90658ad5a300da53b43a28fe991808fb4006   jul  2021     auto   \n",
       "1225  fffd2d78fc884b609293e46bfc90641926869ebb   jul  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto        Tucson            789.124172   \n",
       "1             hold        Tucson            777.244898   \n",
       "2             auto        Tucson            762.216667   \n",
       "3             hold        Tucson            765.257048   \n",
       "4             hold        Peoria            767.103746   \n",
       "...            ...           ...                   ...   \n",
       "1221          hold       Gilbert            772.022325   \n",
       "1222          hold       Phoenix            794.257252   \n",
       "1223          hold       Phoenix            797.165756   \n",
       "1224          hold  Sierra Vista            752.119628   \n",
       "1225          hold    Scottsdale            750.899824   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      795.200331                   780.149007   \n",
       "1                      777.530612                   777.346939   \n",
       "2                      756.666667                   683.333333   \n",
       "3                      762.363184                   676.230514   \n",
       "4                      767.233429                   766.662824   \n",
       "...                           ...                          ...   \n",
       "1221                   772.220725                   709.993682   \n",
       "1222                   798.941221                   786.237405   \n",
       "1223                   794.816029                   783.870674   \n",
       "1224                   752.822300                   680.000000   \n",
       "1225                   749.143234                   749.118629   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    10.0             False         False            False  \n",
       "1                    10.0             False         False            False  \n",
       "2                     0.0             False         False             True  \n",
       "3                     0.0             False         False             True  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1221                  9.0             False         False            False  \n",
       "1222                  0.0             False         False            False  \n",
       "1223                 50.0              True         False             True  \n",
       "1224                  0.0             False         False            False  \n",
       "1225                 30.0             False         False            False  \n",
       "\n",
       "[9181 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/jul/\" + file)\n",
    "    AZ_jul = pd.concat([AZ_jul, df])\n",
    "    \n",
    "AZ_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_jul.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/AZ-day/2017-aug-day-AZ.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b4a23bcbdeda88fe06692dbac3148deaa9871e95</td>\n",
       "      <td>2017-08-20 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>790</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6818c05b83307ffaabd0fa8e3f81df9d7afd6d40</td>\n",
       "      <td>2017-08-09 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f432001783f1362a57e97ef3a959e6e3d42b0a70</td>\n",
       "      <td>2017-08-12 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>777</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80cd08f9d15226657414f467a0a84c8d6fe9994e</td>\n",
       "      <td>2017-08-13 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>789</td>\n",
       "      <td>785</td>\n",
       "      <td>785</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b8a96b5d5de04f18d98fd153368b3469c03ba654</td>\n",
       "      <td>2017-08-19 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>797</td>\n",
       "      <td>667</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653621</th>\n",
       "      <td>eea1a8dd2b5f5a31877d9be93f05d3069dd79f2b</td>\n",
       "      <td>2017-08-16 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653622</th>\n",
       "      <td>3355007b4db5b4b7812ef09d71c3df920c4d99a7</td>\n",
       "      <td>2017-08-11 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653623</th>\n",
       "      <td>8569729795b5c3d9a5165c301651884d5a3b1311</td>\n",
       "      <td>2017-08-07 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>778</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653624</th>\n",
       "      <td>b69d1addc33fab4713d505e909d7871235d66939</td>\n",
       "      <td>2017-08-14 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653625</th>\n",
       "      <td>db5a4476a4db124dd15ddee184e7383ddad6701c</td>\n",
       "      <td>2017-08-05 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>624019 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b4a23bcbdeda88fe06692dbac3148deaa9871e95  2017-08-20 19:35:00 UTC   \n",
       "1       6818c05b83307ffaabd0fa8e3f81df9d7afd6d40  2017-08-09 16:10:00 UTC   \n",
       "2       f432001783f1362a57e97ef3a959e6e3d42b0a70  2017-08-12 16:40:00 UTC   \n",
       "3       80cd08f9d15226657414f467a0a84c8d6fe9994e  2017-08-13 14:05:00 UTC   \n",
       "4       b8a96b5d5de04f18d98fd153368b3469c03ba654  2017-08-19 16:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "653621  eea1a8dd2b5f5a31877d9be93f05d3069dd79f2b  2017-08-16 15:55:00 UTC   \n",
       "653622  3355007b4db5b4b7812ef09d71c3df920c4d99a7  2017-08-11 13:40:00 UTC   \n",
       "653623  8569729795b5c3d9a5165c301651884d5a3b1311  2017-08-07 17:05:00 UTC   \n",
       "653624  b69d1addc33fab4713d505e909d7871235d66939  2017-08-14 15:25:00 UTC   \n",
       "653625  db5a4476a4db124dd15ddee184e7383ddad6701c  2017-08-05 18:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               790                      785   \n",
       "1          cool          hold               683                      705   \n",
       "2          cool          hold               777                      765   \n",
       "3          cool          hold               789                      785   \n",
       "4          cool          auto               762                      797   \n",
       "...         ...           ...               ...                      ...   \n",
       "653621     cool          auto               756                      760   \n",
       "653622     cool          hold               758                      760   \n",
       "653623     cool          auto               778                      780   \n",
       "653624     cool          hold               751                      760   \n",
       "653625     cool          auto               774                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           735            AZ     Phoenix   \n",
       "1                           705            AZ     Phoenix   \n",
       "2                           765            AZ     Phoenix   \n",
       "3                           785            AZ     Phoenix   \n",
       "4                           667            AZ  Scottsdale   \n",
       "...                         ...           ...         ...   \n",
       "653621                      760            AZ     Buckeye   \n",
       "653622                      760            AZ  Scottsdale   \n",
       "653623                      760            AZ      Peoria   \n",
       "653624                      760            AZ  Cave Creek   \n",
       "653625                      760            AZ        Mesa   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        70             False         False             True   \n",
       "2                        15             False         False            False   \n",
       "3                        15             False         False            False   \n",
       "4                        15             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "653621                   10             False         False             True   \n",
       "653622                    5             False         False             True   \n",
       "653623                    0             False         False            False   \n",
       "653624                    0              True         False            False   \n",
       "653625                   20             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "653621                  Electric  \n",
       "653622                  Electric  \n",
       "653623                       Gas  \n",
       "653624                       Gas  \n",
       "653625                  Electric  \n",
       "\n",
       "[624019 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/AZ/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/AZ-day/2018-aug-day-AZ.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1f539e5f414bc936f626b439b92663afd5c1c0dc</td>\n",
       "      <td>2018-08-11 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>776</td>\n",
       "      <td>780</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40fe0ad2c29d4fa1f16443a2d1ce30642276805a</td>\n",
       "      <td>2018-08-08 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>777</td>\n",
       "      <td>789</td>\n",
       "      <td>789</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dcbc9be1ef503fcca9989bf30e24f0bce0688d33</td>\n",
       "      <td>2018-08-04 07:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>665</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2f1441ae5ba99f821d7ec9eeac1255d41e14827d</td>\n",
       "      <td>2018-08-09 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1d6bdc097e50dcf69a5d14a9fb29fd9cf37e9daf</td>\n",
       "      <td>2018-08-30 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>798</td>\n",
       "      <td>804</td>\n",
       "      <td>696</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085032</th>\n",
       "      <td>ca5cca779687803d7140569bb2b539e113d235a1</td>\n",
       "      <td>2018-08-25 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Prescott Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085033</th>\n",
       "      <td>8fcf121e649ebf697caa49b3795602008c137558</td>\n",
       "      <td>2018-08-26 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085034</th>\n",
       "      <td>65e9024cbb0e78abb70a28b31c1ef32c92635b0b</td>\n",
       "      <td>2018-08-20 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085035</th>\n",
       "      <td>5f37a90ab75318f95f477eceeea7997fbecfe523</td>\n",
       "      <td>2018-08-10 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1085036</th>\n",
       "      <td>336eb8b88b82e199460fa27a7e2dd0628d9b268d</td>\n",
       "      <td>2018-08-20 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>772</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Buckeye</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1024021 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        1f539e5f414bc936f626b439b92663afd5c1c0dc  2018-08-11 15:40:00 UTC   \n",
       "1        40fe0ad2c29d4fa1f16443a2d1ce30642276805a  2018-08-08 19:15:00 UTC   \n",
       "3        dcbc9be1ef503fcca9989bf30e24f0bce0688d33  2018-08-04 07:55:00 UTC   \n",
       "5        2f1441ae5ba99f821d7ec9eeac1255d41e14827d  2018-08-09 14:10:00 UTC   \n",
       "6        1d6bdc097e50dcf69a5d14a9fb29fd9cf37e9daf  2018-08-30 18:50:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1085032  ca5cca779687803d7140569bb2b539e113d235a1  2018-08-25 19:15:00 UTC   \n",
       "1085033  8fcf121e649ebf697caa49b3795602008c137558  2018-08-26 16:15:00 UTC   \n",
       "1085034  65e9024cbb0e78abb70a28b31c1ef32c92635b0b  2018-08-20 16:25:00 UTC   \n",
       "1085035  5f37a90ab75318f95f477eceeea7997fbecfe523  2018-08-10 17:35:00 UTC   \n",
       "1085036  336eb8b88b82e199460fa27a7e2dd0628d9b268d  2018-08-20 13:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               776                      780   \n",
       "1           cool          hold               777                      789   \n",
       "3           cool          hold               742                      665   \n",
       "5           cool          hold               721                      729   \n",
       "6           auto          hold               798                      804   \n",
       "...          ...           ...               ...                      ...   \n",
       "1085032     cool          hold               756                      760   \n",
       "1085033     cool          hold               759                      760   \n",
       "1085034     cool          hold               755                      760   \n",
       "1085035     cool          hold               756                      760   \n",
       "1085036     cool          auto               772                      780   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            715            AZ         Chandler   \n",
       "1                            789            AZ          Phoenix   \n",
       "3                            665            AZ       Scottsdale   \n",
       "5                            729            AZ          Phoenix   \n",
       "6                            696            AZ          Gilbert   \n",
       "...                          ...           ...              ...   \n",
       "1085032                      760            AZ  Prescott Valley   \n",
       "1085033                      760            AZ           Tucson   \n",
       "1085034                      760            AZ           Tucson   \n",
       "1085035                      760            AZ            Tempe   \n",
       "1085036                      760            AZ          Buckeye   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         15             False         False            False   \n",
       "1                          0              True         False            False   \n",
       "3                          0              True         False             True   \n",
       "5                         39              True         False             True   \n",
       "6                          6             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1085032                   10             False         False            False   \n",
       "1085033                   10             False         False            False   \n",
       "1085034                   15              True         False            False   \n",
       "1085035                   25             False         False             True   \n",
       "1085036                   15             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "3                        Electric  \n",
       "5                        Electric  \n",
       "6                             Gas  \n",
       "...                           ...  \n",
       "1085032                       Gas  \n",
       "1085033                       Gas  \n",
       "1085034                       Gas  \n",
       "1085035                  Electric  \n",
       "1085036                       Gas  \n",
       "\n",
       "[1024021 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/AZ/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/AZ-day/2019-aug-day-AZ.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>615966cb5d0c6b63c9687da35e01a7df502713ca</td>\n",
       "      <td>2019-08-20 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>782</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b0c61cfce30760006ad4e6375155c98fd14255b1</td>\n",
       "      <td>2019-08-13 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ca5cca779687803d7140569bb2b539e113d235a1</td>\n",
       "      <td>2019-08-18 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>764</td>\n",
       "      <td>764</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Prescott Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>82cec3808a8aec4c1d3a83ac95de458d4a1806e0</td>\n",
       "      <td>2019-08-17 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>712</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Queen Creek</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12662f413188b006f2769990f117b74c518eca82</td>\n",
       "      <td>2019-08-13 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488340</th>\n",
       "      <td>a55a645c8eab67fc47ab4926ca485c8f3210d951</td>\n",
       "      <td>2019-08-11 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Goodyear</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488341</th>\n",
       "      <td>a020371cea6e8a05029f83acce0591a487ed3be5</td>\n",
       "      <td>2019-08-21 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488342</th>\n",
       "      <td>efebef567ff3dd92fbc5816fa35ec010c08de832</td>\n",
       "      <td>2019-08-09 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488343</th>\n",
       "      <td>f1df213f9d1456838cd95bd588fcb5591ebc66ae</td>\n",
       "      <td>2019-08-04 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Rio Verde</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1488344</th>\n",
       "      <td>3a57f62536b559d92d79a92cb351676c0fdc98ba</td>\n",
       "      <td>2019-08-28 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1388862 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        615966cb5d0c6b63c9687da35e01a7df502713ca  2019-08-20 15:10:00 UTC   \n",
       "2        b0c61cfce30760006ad4e6375155c98fd14255b1  2019-08-13 13:30:00 UTC   \n",
       "3        ca5cca779687803d7140569bb2b539e113d235a1  2019-08-18 14:50:00 UTC   \n",
       "5        82cec3808a8aec4c1d3a83ac95de458d4a1806e0  2019-08-17 19:40:00 UTC   \n",
       "6        12662f413188b006f2769990f117b74c518eca82  2019-08-13 17:05:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1488340  a55a645c8eab67fc47ab4926ca485c8f3210d951  2019-08-11 12:05:00 UTC   \n",
       "1488341  a020371cea6e8a05029f83acce0591a487ed3be5  2019-08-21 18:35:00 UTC   \n",
       "1488342  efebef567ff3dd92fbc5816fa35ec010c08de832  2019-08-09 14:00:00 UTC   \n",
       "1488343  f1df213f9d1456838cd95bd588fcb5591ebc66ae  2019-08-04 13:55:00 UTC   \n",
       "1488344  3a57f62536b559d92d79a92cb351676c0fdc98ba  2019-08-28 14:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           auto          auto               782                      785   \n",
       "2           cool          hold               725                      725   \n",
       "3           cool          hold               768                      764   \n",
       "5           auto          hold               762                      762   \n",
       "6           cool          hold               753                      745   \n",
       "...          ...           ...               ...                      ...   \n",
       "1488340     cool          auto               758                      760   \n",
       "1488341     cool          hold               756                      760   \n",
       "1488342     cool          hold               759                      760   \n",
       "1488343     cool          hold               763                      760   \n",
       "1488344     cool          hold               766                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "1                            735            AZ           Tucson   \n",
       "2                            725            AZ           Tucson   \n",
       "3                            764            AZ  Prescott Valley   \n",
       "5                            712            AZ      Queen Creek   \n",
       "6                            745            AZ             Mesa   \n",
       "...                          ...           ...              ...   \n",
       "1488340                      760            AZ         Goodyear   \n",
       "1488341                      760            AZ             Mesa   \n",
       "1488342                      760            AZ          Phoenix   \n",
       "1488343                      760            AZ        Rio Verde   \n",
       "1488344                      760            AZ          Phoenix   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         69             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         10             False         False            False   \n",
       "5                          0             False         False            False   \n",
       "6                         35             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1488340                    9             False         False            False   \n",
       "1488341                    0             False         False             True   \n",
       "1488342                   10             False         False            False   \n",
       "1488343                    0             False         False             True   \n",
       "1488344                   25             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "5                             Gas  \n",
       "6                        Electric  \n",
       "...                           ...  \n",
       "1488340                       Gas  \n",
       "1488341                  Electric  \n",
       "1488342                       Gas  \n",
       "1488343                  Electric  \n",
       "1488344                  Electric  \n",
       "\n",
       "[1388862 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/AZ/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/AZ-day/2020-aug-day-AZ.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52e92d3957712db8fd805073e024c90a95817381</td>\n",
       "      <td>2020-08-24 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>745</td>\n",
       "      <td>695</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>425ac3626cf6e1ad590f194d5e1f78fba02b16cd</td>\n",
       "      <td>2020-08-26 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>792</td>\n",
       "      <td>790</td>\n",
       "      <td>748</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cbff3b7a1a08ff17cfc7b97d6b3eaa0461148f51</td>\n",
       "      <td>2020-08-24 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>777</td>\n",
       "      <td>780</td>\n",
       "      <td>749</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a41499894359eb09406fab6c64e2d9dd7f9cd252</td>\n",
       "      <td>2020-08-26 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1f40eb23680da761bcf80f94387541f9241a9f72</td>\n",
       "      <td>2020-08-18 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1534300</th>\n",
       "      <td>341ded0d08168fcb6d28b91d7640f936dfa91d38</td>\n",
       "      <td>2020-08-25 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>glendale</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1534301</th>\n",
       "      <td>0e04bc911366aef7ce21b65505d1f6015b181d5a</td>\n",
       "      <td>2020-08-25 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1534302</th>\n",
       "      <td>c6962c2b095020186e2df5390f5f0e779c0f6e36</td>\n",
       "      <td>2020-08-04 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1534303</th>\n",
       "      <td>bb1191aa00ff520c09a63cecc92aba1bf44072d1</td>\n",
       "      <td>2020-08-06 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Gilbert</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1534304</th>\n",
       "      <td>21d77c12474755ea59246f01b7431ef5f05fb46c</td>\n",
       "      <td>2020-08-15 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Anthem</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1463706 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        52e92d3957712db8fd805073e024c90a95817381  2020-08-24 15:00:00 UTC   \n",
       "1        425ac3626cf6e1ad590f194d5e1f78fba02b16cd  2020-08-26 18:45:00 UTC   \n",
       "2        cbff3b7a1a08ff17cfc7b97d6b3eaa0461148f51  2020-08-24 17:00:00 UTC   \n",
       "3        a41499894359eb09406fab6c64e2d9dd7f9cd252  2020-08-26 16:15:00 UTC   \n",
       "4        1f40eb23680da761bcf80f94387541f9241a9f72  2020-08-18 15:05:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1534300  341ded0d08168fcb6d28b91d7640f936dfa91d38  2020-08-25 17:55:00 UTC   \n",
       "1534301  0e04bc911366aef7ce21b65505d1f6015b181d5a  2020-08-25 16:15:00 UTC   \n",
       "1534302  c6962c2b095020186e2df5390f5f0e779c0f6e36  2020-08-04 19:05:00 UTC   \n",
       "1534303  bb1191aa00ff520c09a63cecc92aba1bf44072d1  2020-08-06 15:05:00 UTC   \n",
       "1534304  21d77c12474755ea59246f01b7431ef5f05fb46c  2020-08-15 19:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               755                      745   \n",
       "1           cool          auto               792                      790   \n",
       "2           cool          auto               777                      780   \n",
       "3           cool          hold               766                      765   \n",
       "4           cool          hold               705                      698   \n",
       "...          ...           ...               ...                      ...   \n",
       "1534300     cool          hold               760                      760   \n",
       "1534301     cool          hold               761                      760   \n",
       "1534302     cool          auto               790                      760   \n",
       "1534303     cool          hold               759                      760   \n",
       "1534304     cool          hold               762                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                            695            AZ     Phoenix   \n",
       "1                            748            AZ    Chandler   \n",
       "2                            749            AZ  Scottsdale   \n",
       "3                            765            AZ        Mesa   \n",
       "4                            698            AZ  Scottsdale   \n",
       "...                          ...           ...         ...   \n",
       "1534300                      760            AZ   glendale    \n",
       "1534301                      760            AZ     Phoenix   \n",
       "1534302                      760            AZ    Chandler   \n",
       "1534303                      760            AZ     Gilbert   \n",
       "1534304                      760            AZ      Anthem   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         50             False         False             True   \n",
       "1                         10              True         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          0             False         False             True   \n",
       "4                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1534300                   20             False         False            False   \n",
       "1534301                   35              True         False             True   \n",
       "1534302                    0             False         False            False   \n",
       "1534303                    5             False         False            False   \n",
       "1534304                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1534300                       Gas  \n",
       "1534301                  Electric  \n",
       "1534302                       Gas  \n",
       "1534303                       Gas  \n",
       "1534304                       Gas  \n",
       "\n",
       "[1463706 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/AZ/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>785.218147</td>\n",
       "      <td>796.067568</td>\n",
       "      <td>778.753861</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>781.499365</td>\n",
       "      <td>787.621347</td>\n",
       "      <td>781.158831</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>752.400000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>752.829225</td>\n",
       "      <td>749.693662</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01016c682a7be3fa75027c0ebd9edf413cf2459e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>777.000000</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2289</th>\n",
       "      <td>ff3e086b5016e9175c0025a60b8f6db90bb1ef6e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Mohave</td>\n",
       "      <td>760.490566</td>\n",
       "      <td>753.600000</td>\n",
       "      <td>685.196226</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2290</th>\n",
       "      <td>ffc69aa005f871624e7187d94b86f1d795252fd7</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>815.188772</td>\n",
       "      <td>811.806909</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2291</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>766.198135</td>\n",
       "      <td>767.090909</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2292</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>759.710340</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2293</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>756.673780</td>\n",
       "      <td>752.577744</td>\n",
       "      <td>752.390244</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7641 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00ab105be10ba80cca71a5e8d83088116fe17003   aug  2017     cool   \n",
       "1     00ab105be10ba80cca71a5e8d83088116fe17003   aug  2017     cool   \n",
       "2     00abe73a91e4a0012069878eba34b166551cce4d   aug  2017     auto   \n",
       "3     00abe73a91e4a0012069878eba34b166551cce4d   aug  2017     auto   \n",
       "4     01016c682a7be3fa75027c0ebd9edf413cf2459e   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2289  ff3e086b5016e9175c0025a60b8f6db90bb1ef6e   aug  2020     auto   \n",
       "2290  ffc69aa005f871624e7187d94b86f1d795252fd7   aug  2020     cool   \n",
       "2291  ffec90658ad5a300da53b43a28fe991808fb4006   aug  2020     auto   \n",
       "2292  ffec90658ad5a300da53b43a28fe991808fb4006   aug  2020     auto   \n",
       "2293  fffd2d78fc884b609293e46bfc90641926869ebb   aug  2020     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto        Tucson            785.218147   \n",
       "1             hold        Tucson            781.499365   \n",
       "2             auto        Tucson            752.400000   \n",
       "3             hold        Tucson            752.829225   \n",
       "4             hold        Peoria            777.000000   \n",
       "...            ...           ...                   ...   \n",
       "2289          hold   Fort Mohave            760.490566   \n",
       "2290          hold       Phoenix            815.188772   \n",
       "2291          auto  Sierra Vista            766.198135   \n",
       "2292          hold  Sierra Vista            759.710340   \n",
       "2293          hold    Scottsdale            756.673780   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      796.067568                   778.753861   \n",
       "1                      787.621347                   781.158831   \n",
       "2                      750.000000                   680.000000   \n",
       "3                      749.693662                   690.000000   \n",
       "4                      770.000000                   770.000000   \n",
       "...                           ...                          ...   \n",
       "2289                   753.600000                   685.196226   \n",
       "2290                   811.806909                   790.000000   \n",
       "2291                   767.090909                   690.000000   \n",
       "2292                   760.000000                   690.000000   \n",
       "2293                   752.577744                   752.390244   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    10.0             False         False            False  \n",
       "1                    10.0             False         False            False  \n",
       "2                     0.0             False         False             True  \n",
       "3                     0.0             False         False             True  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2289                 25.0             False         False            False  \n",
       "2290                 50.0              True         False             True  \n",
       "2291                  0.0             False         False            False  \n",
       "2292                  0.0             False         False            False  \n",
       "2293                 30.0             False         False            False  \n",
       "\n",
       "[7641 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/aug/\" + file)\n",
    "    AZ_aug = pd.concat([AZ_aug, df])\n",
    "    \n",
    "AZ_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_aug.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/AZ-day/2017-dec-day-AZ.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7fdbd7a865e15e8e491e6750ee9c4117ce9847d7</td>\n",
       "      <td>2017-12-22 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>666</td>\n",
       "      <td>754</td>\n",
       "      <td>686</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b5fe7a5af6568e6a574f3868c8da0399e656e27d</td>\n",
       "      <td>2017-12-12 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Vail</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14936a3e0bf350ef87ccaddad0014c7a4b51da2a</td>\n",
       "      <td>2017-12-30 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c90b6b66871e2d1835134af14e3d01688d94fd45</td>\n",
       "      <td>2017-12-14 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6820a043eade62efd830673676f29040e1310722</td>\n",
       "      <td>2017-12-10 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>741</td>\n",
       "      <td>741</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Cave Creek</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711486</th>\n",
       "      <td>86e7e6e7c2079a537bba14707aba2d945ebbefaf</td>\n",
       "      <td>2017-12-15 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>765</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711487</th>\n",
       "      <td>2623afd8550dd9ae290a7d6d2f63179d36ecdc9a</td>\n",
       "      <td>2017-12-09 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711488</th>\n",
       "      <td>66a0c5209e2c4c317b9e858613671086fed7f17f</td>\n",
       "      <td>2017-12-29 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Queen Creek</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711489</th>\n",
       "      <td>86e7e6e7c2079a537bba14707aba2d945ebbefaf</td>\n",
       "      <td>2017-12-31 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>765</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>711490</th>\n",
       "      <td>2623afd8550dd9ae290a7d6d2f63179d36ecdc9a</td>\n",
       "      <td>2017-12-12 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>672850 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7fdbd7a865e15e8e491e6750ee9c4117ce9847d7  2017-12-22 15:15:00 UTC   \n",
       "1       b5fe7a5af6568e6a574f3868c8da0399e656e27d  2017-12-12 17:10:00 UTC   \n",
       "3       14936a3e0bf350ef87ccaddad0014c7a4b51da2a  2017-12-30 19:35:00 UTC   \n",
       "4       c90b6b66871e2d1835134af14e3d01688d94fd45  2017-12-14 12:35:00 UTC   \n",
       "6       6820a043eade62efd830673676f29040e1310722  2017-12-10 18:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "711486  86e7e6e7c2079a537bba14707aba2d945ebbefaf  2017-12-15 19:35:00 UTC   \n",
       "711487  2623afd8550dd9ae290a7d6d2f63179d36ecdc9a  2017-12-09 19:05:00 UTC   \n",
       "711488  66a0c5209e2c4c317b9e858613671086fed7f17f  2017-12-29 17:25:00 UTC   \n",
       "711489  86e7e6e7c2079a537bba14707aba2d945ebbefaf  2017-12-31 14:55:00 UTC   \n",
       "711490  2623afd8550dd9ae290a7d6d2f63179d36ecdc9a  2017-12-12 14:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               666                      754   \n",
       "1          heat          hold               760                      755   \n",
       "3          cool          auto               683                      695   \n",
       "4          auto          hold               724                      755   \n",
       "6          heat          hold               709                      741   \n",
       "...         ...           ...               ...                      ...   \n",
       "711486     auto          hold               693                      765   \n",
       "711487     auto          auto               717                      765   \n",
       "711488     auto          auto               713                      765   \n",
       "711489     auto          hold               653                      765   \n",
       "711490     auto          auto               715                      765   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           686            AZ      Phoenix   \n",
       "1                           755            AZ         Vail   \n",
       "3                           645            AZ   Cave Creek   \n",
       "4                           705            AZ    Sahuarita   \n",
       "6                           741            AZ   Cave Creek   \n",
       "...                         ...           ...          ...   \n",
       "711486                      665            AZ      Phoenix   \n",
       "711487                      715            AZ       Tucson   \n",
       "711488                      715            AZ  Queen Creek   \n",
       "711489                      665            AZ      Phoenix   \n",
       "711490                      715            AZ       Tucson   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False             True   \n",
       "1                        15             False         False            False   \n",
       "3                        27             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "6                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "711486                   40              True         False             True   \n",
       "711487                    0             False         False             True   \n",
       "711488                   10             False         False            False   \n",
       "711489                   40              True         False             True   \n",
       "711490                    0             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "711486                  Electric  \n",
       "711487                  Electric  \n",
       "711488                       Gas  \n",
       "711489                  Electric  \n",
       "711490                  Electric  \n",
       "\n",
       "[672850 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/AZ/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/AZ-day/2018-dec-day-AZ.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17e47567ce0de68562ebe6ba0aca97d53c37c32f</td>\n",
       "      <td>2018-12-31 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Flagstaff</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>42625b8bceff1a433465301087b67df701c4c895</td>\n",
       "      <td>2018-12-19 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>713</td>\n",
       "      <td>713</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tolleson</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2d9bff633934542b154b6470e733831a6ab9e239</td>\n",
       "      <td>2018-12-15 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>756</td>\n",
       "      <td>756</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>92ddb2b34ea659d91151d9f1a781b7132949706a</td>\n",
       "      <td>2018-12-17 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8b814b0f15283f86cace65975e000334c433eb2c</td>\n",
       "      <td>2018-12-29 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sahuarita</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034291</th>\n",
       "      <td>9dcd6d16fdc31ab22ec8e482574f92319b29aabb</td>\n",
       "      <td>2018-12-06 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034292</th>\n",
       "      <td>31f15af8291f3fb2d23d27f28cae917c4adbb415</td>\n",
       "      <td>2018-12-08 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034293</th>\n",
       "      <td>2dd19cb5f1aab8aa67ca2411d9710c287b675bc3</td>\n",
       "      <td>2018-12-25 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Maricopa</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034294</th>\n",
       "      <td>3eaad16e681de383b78d5cbbc7bbb4f5747a710b</td>\n",
       "      <td>2018-12-03 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1034295</th>\n",
       "      <td>71dfa478a749c91a51ecd00f33f9824ad982aeb2</td>\n",
       "      <td>2018-12-23 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tempe</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>986318 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        17e47567ce0de68562ebe6ba0aca97d53c37c32f  2018-12-31 18:05:00 UTC   \n",
       "1        42625b8bceff1a433465301087b67df701c4c895  2018-12-19 14:00:00 UTC   \n",
       "2        2d9bff633934542b154b6470e733831a6ab9e239  2018-12-15 18:50:00 UTC   \n",
       "3        92ddb2b34ea659d91151d9f1a781b7132949706a  2018-12-17 14:15:00 UTC   \n",
       "4        8b814b0f15283f86cace65975e000334c433eb2c  2018-12-29 16:05:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1034291  9dcd6d16fdc31ab22ec8e482574f92319b29aabb  2018-12-06 13:40:00 UTC   \n",
       "1034292  31f15af8291f3fb2d23d27f28cae917c4adbb415  2018-12-08 13:50:00 UTC   \n",
       "1034293  2dd19cb5f1aab8aa67ca2411d9710c287b675bc3  2018-12-25 19:55:00 UTC   \n",
       "1034294  3eaad16e681de383b78d5cbbc7bbb4f5747a710b  2018-12-03 17:20:00 UTC   \n",
       "1034295  71dfa478a749c91a51ecd00f33f9824ad982aeb2  2018-12-23 18:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               701                      705   \n",
       "1           heat          hold               712                      713   \n",
       "2           heat          hold               766                      756   \n",
       "3           heat          hold               720                      719   \n",
       "4           heat          hold               695                      689   \n",
       "...          ...           ...               ...                      ...   \n",
       "1034291     heat          auto               767                      760   \n",
       "1034292     auto          hold               691                      760   \n",
       "1034293     auto          auto               692                      760   \n",
       "1034294     auto          hold               689                      760   \n",
       "1034295     auto          hold               678                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState       City  \\\n",
       "0                            705            AZ  Flagstaff   \n",
       "1                            713            AZ   Tolleson   \n",
       "2                            756            AZ       Mesa   \n",
       "3                            719            AZ       Mesa   \n",
       "4                            689            AZ  Sahuarita   \n",
       "...                          ...           ...        ...   \n",
       "1034291                      760            AZ    Phoenix   \n",
       "1034292                      690            AZ     Tucson   \n",
       "1034293                      680            AZ   Maricopa   \n",
       "1034294                      690            AZ   Surprise   \n",
       "1034295                      680            AZ      Tempe   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         15              True         False             True   \n",
       "2                          5             False         False            False   \n",
       "3                          0             False         False            False   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1034291                    0             False         False             True   \n",
       "1034292                   45             False         False            False   \n",
       "1034293                   10             False         False            False   \n",
       "1034294                   15             False         False            False   \n",
       "1034295                   45             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1034291                  Electric  \n",
       "1034292                       Gas  \n",
       "1034293                       Gas  \n",
       "1034294                       Gas  \n",
       "1034295                       Gas  \n",
       "\n",
       "[986318 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/AZ/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/AZ-day/2019-dec-day-AZ.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0222630f87433cb6a60e0fe83c9bc2233a842fce</td>\n",
       "      <td>2019-12-05 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4e8f298070a01421704c96522699fcb8831e9626</td>\n",
       "      <td>2019-12-31 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>761</td>\n",
       "      <td>671</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>52efa58814c9787b1ead3595ff9082a24aa021cd</td>\n",
       "      <td>2019-12-04 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>718</td>\n",
       "      <td>700</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2cdee45cabae6d78ac89cea0d28a49dd931f1950</td>\n",
       "      <td>2019-12-18 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>630</td>\n",
       "      <td>825</td>\n",
       "      <td>630</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3bb59c839fe174965abfaf5948e014193baa054a</td>\n",
       "      <td>2019-12-15 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>715</td>\n",
       "      <td>665</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1336147</th>\n",
       "      <td>8b8abd0573b58905f7af953cc6e9c8d8d05f40d4</td>\n",
       "      <td>2019-12-20 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1336148</th>\n",
       "      <td>8b8abd0573b58905f7af953cc6e9c8d8d05f40d4</td>\n",
       "      <td>2019-12-20 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1336149</th>\n",
       "      <td>a1fb3a3144cb137c49b18bbadd05fa086c0755e8</td>\n",
       "      <td>2019-12-20 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1336150</th>\n",
       "      <td>a41499894359eb09406fab6c64e2d9dd7f9cd252</td>\n",
       "      <td>2019-12-28 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Mesa</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1336151</th>\n",
       "      <td>8b4090a877444b902965cecbc69396cc72ad4df7</td>\n",
       "      <td>2019-12-24 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>765</td>\n",
       "      <td>685</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1252349 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        0222630f87433cb6a60e0fe83c9bc2233a842fce  2019-12-05 14:15:00 UTC   \n",
       "2        4e8f298070a01421704c96522699fcb8831e9626  2019-12-31 14:35:00 UTC   \n",
       "3        52efa58814c9787b1ead3595ff9082a24aa021cd  2019-12-04 14:15:00 UTC   \n",
       "4        2cdee45cabae6d78ac89cea0d28a49dd931f1950  2019-12-18 17:55:00 UTC   \n",
       "5        3bb59c839fe174965abfaf5948e014193baa054a  2019-12-15 14:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1336147  8b8abd0573b58905f7af953cc6e9c8d8d05f40d4  2019-12-20 18:45:00 UTC   \n",
       "1336148  8b8abd0573b58905f7af953cc6e9c8d8d05f40d4  2019-12-20 19:05:00 UTC   \n",
       "1336149  a1fb3a3144cb137c49b18bbadd05fa086c0755e8  2019-12-20 13:00:00 UTC   \n",
       "1336150  a41499894359eb09406fab6c64e2d9dd7f9cd252  2019-12-28 19:45:00 UTC   \n",
       "1336151  8b4090a877444b902965cecbc69396cc72ad4df7  2019-12-24 18:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               749                      715   \n",
       "2           heat          hold               709                      761   \n",
       "3           heat          hold               695                      718   \n",
       "4           auto          auto               630                      825   \n",
       "5           auto          hold               700                      715   \n",
       "...          ...           ...               ...                      ...   \n",
       "1336147     auto          hold               727                      765   \n",
       "1336148     auto          hold               725                      765   \n",
       "1336149     auto          auto               723                      765   \n",
       "1336150     auto          hold               718                      765   \n",
       "1336151     auto          auto               687                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                            715            AZ   Phoenix                    0   \n",
       "2                            671            AZ   Phoenix                   55   \n",
       "3                            700            AZ   Phoenix                   70   \n",
       "4                            630            AZ    Peoria                   19   \n",
       "5                            665            AZ   Phoenix                   20   \n",
       "...                          ...           ...       ...                  ...   \n",
       "1336147                      715            AZ   Phoenix                   60   \n",
       "1336148                      715            AZ   Phoenix                   60   \n",
       "1336149                      715            AZ  Chandler                    0   \n",
       "1336150                      715            AZ      Mesa                    0   \n",
       "1336151                      685            AZ    Peoria                    0   \n",
       "\n",
       "         allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                    True         False            False   \n",
       "2                   False         False             True   \n",
       "3                   False         False            False   \n",
       "4                   False         False            False   \n",
       "5                   False         False             True   \n",
       "...                   ...           ...              ...   \n",
       "1336147              True         False            False   \n",
       "1336148              True         False            False   \n",
       "1336149              True         False            False   \n",
       "1336150             False         False             True   \n",
       "1336151              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "1336147                       Gas  \n",
       "1336148                       Gas  \n",
       "1336149                       Gas  \n",
       "1336150                  Electric  \n",
       "1336151                  Electric  \n",
       "\n",
       "[1252349 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/AZ/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/AZ-day/2020-dec-day-AZ.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8b4090a877444b902965cecbc69396cc72ad4df7</td>\n",
       "      <td>2020-12-31 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>785</td>\n",
       "      <td>625</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ab044de6713db537806cad24400f9ede54faa142</td>\n",
       "      <td>2020-12-26 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>633</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Sun City</td>\n",
       "      <td>70</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36c558f4706dbc66d799758e6e0f082e3395021d</td>\n",
       "      <td>2020-12-03 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>619</td>\n",
       "      <td>840</td>\n",
       "      <td>780</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Chandler</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f66a249909690b7643de98295e65498b72b85ecf</td>\n",
       "      <td>2020-12-26 09:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>759</td>\n",
       "      <td>759</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ddade16a8259d317804b25a867a487bd7a7ced5a</td>\n",
       "      <td>2020-12-20 07:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>833</td>\n",
       "      <td>723</td>\n",
       "      <td>AZ</td>\n",
       "      <td>SCOTTSDALE</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149364</th>\n",
       "      <td>0ac042f6fc949edceac5fb1ecda5a16bc4e45b77</td>\n",
       "      <td>2020-12-16 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>stanfield</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149365</th>\n",
       "      <td>ef00799907125434509b12c9ade2d3f646ae101f</td>\n",
       "      <td>2020-12-22 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>765</td>\n",
       "      <td>675</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149366</th>\n",
       "      <td>ef00799907125434509b12c9ade2d3f646ae101f</td>\n",
       "      <td>2020-12-14 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>765</td>\n",
       "      <td>675</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149367</th>\n",
       "      <td>615966cb5d0c6b63c9687da35e01a7df502713ca</td>\n",
       "      <td>2020-12-18 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>AZ</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1149368</th>\n",
       "      <td>80467f30cce2c321e9e05405960ce92a377f38bc</td>\n",
       "      <td>2020-12-02 09:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>AZ</td>\n",
       "      <td>San Tan Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1097182 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        8b4090a877444b902965cecbc69396cc72ad4df7  2020-12-31 18:30:00 UTC   \n",
       "1        ab044de6713db537806cad24400f9ede54faa142  2020-12-26 14:15:00 UTC   \n",
       "4        36c558f4706dbc66d799758e6e0f082e3395021d  2020-12-03 15:30:00 UTC   \n",
       "5        f66a249909690b7643de98295e65498b72b85ecf  2020-12-26 09:35:00 UTC   \n",
       "7        ddade16a8259d317804b25a867a487bd7a7ced5a  2020-12-20 07:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1149364  0ac042f6fc949edceac5fb1ecda5a16bc4e45b77  2020-12-16 12:05:00 UTC   \n",
       "1149365  ef00799907125434509b12c9ade2d3f646ae101f  2020-12-22 18:45:00 UTC   \n",
       "1149366  ef00799907125434509b12c9ade2d3f646ae101f  2020-12-14 14:40:00 UTC   \n",
       "1149367  615966cb5d0c6b63c9687da35e01a7df502713ca  2020-12-18 17:00:00 UTC   \n",
       "1149368  80467f30cce2c321e9e05405960ce92a377f38bc  2020-12-02 09:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               660                      785   \n",
       "1           heat          hold               633                      640   \n",
       "4           cool          auto               619                      840   \n",
       "5           heat          hold               753                      759   \n",
       "7           auto          hold               725                      833   \n",
       "...          ...           ...               ...                      ...   \n",
       "1149364     heat          hold               761                      765   \n",
       "1149365     heat          hold               718                      765   \n",
       "1149366     auto          hold               697                      765   \n",
       "1149367     auto          auto               726                      765   \n",
       "1149368     heat          hold               766                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            625            AZ          Peoria   \n",
       "1                            640            AZ        Sun City   \n",
       "4                            780            AZ        Chandler   \n",
       "5                            759            AZ         Phoenix   \n",
       "7                            723            AZ      SCOTTSDALE   \n",
       "...                          ...           ...             ...   \n",
       "1149364                      765            AZ       stanfield   \n",
       "1149365                      675            AZ        Surprise   \n",
       "1149366                      675            AZ        Surprise   \n",
       "1149367                      715            AZ          Tucson   \n",
       "1149368                      765            AZ  San Tan Valley   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False             True   \n",
       "1                         70              True         False             True   \n",
       "4                         25             False         False            False   \n",
       "5                         60             False         False            False   \n",
       "7                         35              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1149364                   10              True         False             True   \n",
       "1149365                   19              True         False            False   \n",
       "1149366                   19              True         False            False   \n",
       "1149367                   69             False         False            False   \n",
       "1149368                   10             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "7                        Electric  \n",
       "...                           ...  \n",
       "1149364                  Electric  \n",
       "1149365                       Gas  \n",
       "1149366                       Gas  \n",
       "1149367                       Gas  \n",
       "1149368                       Gas  \n",
       "\n",
       "[1097182 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/AZ/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/AZ/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>002e06a167b6c8ac5b691f3555a28c83623973b1</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>725.176471</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>698.125000</td>\n",
       "      <td>840.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>732.200000</td>\n",
       "      <td>823.200000</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>698.615385</td>\n",
       "      <td>820.000000</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>715.833333</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2316</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>687.376758</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2317</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>740.996269</td>\n",
       "      <td>792.729478</td>\n",
       "      <td>742.680970</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2318</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>743.658120</td>\n",
       "      <td>796.217949</td>\n",
       "      <td>746.112536</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2319</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>737.888889</td>\n",
       "      <td>737.530864</td>\n",
       "      <td>737.530864</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2320</th>\n",
       "      <td>fffe6c24cd5a508c58628472cd5bd31846ea70e4</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>695.000000</td>\n",
       "      <td>704.666667</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8990 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     002e06a167b6c8ac5b691f3555a28c83623973b1   dec  2017     auto   \n",
       "1     00ab105be10ba80cca71a5e8d83088116fe17003   dec  2017     auto   \n",
       "2     00ab105be10ba80cca71a5e8d83088116fe17003   dec  2017     cool   \n",
       "3     00ab105be10ba80cca71a5e8d83088116fe17003   dec  2017     cool   \n",
       "4     00ab105be10ba80cca71a5e8d83088116fe17003   dec  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2316  ffec90658ad5a300da53b43a28fe991808fb4006   dec  2020     auto   \n",
       "2317  fffd2d78fc884b609293e46bfc90641926869ebb   dec  2020     auto   \n",
       "2318  fffd2d78fc884b609293e46bfc90641926869ebb   dec  2020     auto   \n",
       "2319  fffd2d78fc884b609293e46bfc90641926869ebb   dec  2020     heat   \n",
       "2320  fffe6c24cd5a508c58628472cd5bd31846ea70e4   dec  2020     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto       Phoenix            725.176471   \n",
       "1             hold        Tucson            698.125000   \n",
       "2             auto        Tucson            732.200000   \n",
       "3             hold        Tucson            698.615385   \n",
       "4             auto        Tucson            715.833333   \n",
       "...            ...           ...                   ...   \n",
       "2316          hold  Sierra Vista            687.376758   \n",
       "2317          auto    Scottsdale            740.996269   \n",
       "2318          hold    Scottsdale            743.658120   \n",
       "2319          hold    Scottsdale            737.888889   \n",
       "2320          hold       Phoenix            695.000000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      780.000000                   720.000000   \n",
       "1                      840.000000                   700.000000   \n",
       "2                      823.200000                   790.000000   \n",
       "3                      820.000000                   790.000000   \n",
       "4                      720.000000                   720.000000   \n",
       "...                           ...                          ...   \n",
       "2316                   770.000000                   690.000000   \n",
       "2317                   792.729478                   742.680970   \n",
       "2318                   796.217949                   746.112536   \n",
       "2319                   737.530864                   737.530864   \n",
       "2320                   704.666667                   700.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    20.0             False         False             True  \n",
       "1                    10.0             False         False            False  \n",
       "2                    10.0             False         False            False  \n",
       "3                    10.0             False         False            False  \n",
       "4                    10.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2316                  0.0             False         False            False  \n",
       "2317                 30.0             False         False            False  \n",
       "2318                 30.0             False         False            False  \n",
       "2319                 30.0             False         False            False  \n",
       "2320                 10.0              True         False            False  \n",
       "\n",
       "[8990 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/AZ/dec/\" + file)\n",
    "    AZ_dec = pd.concat([AZ_dec, df])\n",
    "    \n",
    "AZ_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_dec.to_csv(\"Scraper_Output/State_Month_Day/AZ/AZ_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/AZ/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>785.218147</td>\n",
       "      <td>796.067568</td>\n",
       "      <td>778.753861</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00ab105be10ba80cca71a5e8d83088116fe17003</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>781.499365</td>\n",
       "      <td>787.621347</td>\n",
       "      <td>781.158831</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>752.400000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00abe73a91e4a0012069878eba34b166551cce4d</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tucson</td>\n",
       "      <td>752.829225</td>\n",
       "      <td>749.693662</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01016c682a7be3fa75027c0ebd9edf413cf2459e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Peoria</td>\n",
       "      <td>777.000000</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>770.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8793</th>\n",
       "      <td>fe7d60cb7fe777a80cd29f1df2350728a8ac3381</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Surprise</td>\n",
       "      <td>747.600000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8794</th>\n",
       "      <td>ff2a7e846baca4fde6914b30a69e4f6327d76db5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>784.859122</td>\n",
       "      <td>788.730947</td>\n",
       "      <td>783.519630</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8795</th>\n",
       "      <td>ffc69aa005f871624e7187d94b86f1d795252fd7</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phoenix</td>\n",
       "      <td>794.774709</td>\n",
       "      <td>793.793605</td>\n",
       "      <td>786.513081</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8796</th>\n",
       "      <td>ffec90658ad5a300da53b43a28fe991808fb4006</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Sierra Vista</td>\n",
       "      <td>756.847215</td>\n",
       "      <td>757.487925</td>\n",
       "      <td>685.766387</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8797</th>\n",
       "      <td>fffd2d78fc884b609293e46bfc90641926869ebb</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Scottsdale</td>\n",
       "      <td>748.553942</td>\n",
       "      <td>747.784232</td>\n",
       "      <td>747.522822</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50137 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00ab105be10ba80cca71a5e8d83088116fe17003   aug  2017     cool   \n",
       "1     00ab105be10ba80cca71a5e8d83088116fe17003   aug  2017     cool   \n",
       "2     00abe73a91e4a0012069878eba34b166551cce4d   aug  2017     auto   \n",
       "3     00abe73a91e4a0012069878eba34b166551cce4d   aug  2017     auto   \n",
       "4     01016c682a7be3fa75027c0ebd9edf413cf2459e   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "8793  fe7d60cb7fe777a80cd29f1df2350728a8ac3381   jun  2021     cool   \n",
       "8794  ff2a7e846baca4fde6914b30a69e4f6327d76db5   jun  2021     cool   \n",
       "8795  ffc69aa005f871624e7187d94b86f1d795252fd7   jun  2021     cool   \n",
       "8796  ffec90658ad5a300da53b43a28fe991808fb4006   jun  2021     auto   \n",
       "8797  fffd2d78fc884b609293e46bfc90641926869ebb   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto        Tucson            785.218147   \n",
       "1             hold        Tucson            781.499365   \n",
       "2             auto        Tucson            752.400000   \n",
       "3             hold        Tucson            752.829225   \n",
       "4             hold        Peoria            777.000000   \n",
       "...            ...           ...                   ...   \n",
       "8793          hold      Surprise            747.600000   \n",
       "8794          hold       Phoenix            784.859122   \n",
       "8795          hold       Phoenix            794.774709   \n",
       "8796          hold  Sierra Vista            756.847215   \n",
       "8797          hold    Scottsdale            748.553942   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      796.067568                   778.753861   \n",
       "1                      787.621347                   781.158831   \n",
       "2                      750.000000                   680.000000   \n",
       "3                      749.693662                   690.000000   \n",
       "4                      770.000000                   770.000000   \n",
       "...                           ...                          ...   \n",
       "8793                   750.000000                   750.000000   \n",
       "8794                   788.730947                   783.519630   \n",
       "8795                   793.793605                   786.513081   \n",
       "8796                   757.487925                   685.766387   \n",
       "8797                   747.784232                   747.522822   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    10.0             False         False            False  \n",
       "1                    10.0             False         False            False  \n",
       "2                     0.0             False         False             True  \n",
       "3                     0.0             False         False             True  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "8793                  0.0             False         False            False  \n",
       "8794                  0.0             False         False            False  \n",
       "8795                 50.0              True         False             True  \n",
       "8796                  0.0             False         False            False  \n",
       "8797                 30.0             False         False            False  \n",
       "\n",
       "[50137 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "AZ_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/AZ/\" + file)\n",
    "    AZ_all = pd.concat([AZ_all, df])\n",
    "    \n",
    "AZ_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "AZ_all.to_csv(\"Scraper_Output/State_Month_Day/AZ_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['AZ']\n",
      "Unique jan_2018: ['AZ']\n",
      "Unique jan_2019: ['AZ']\n",
      "Unique jan_2020: ['AZ']\n",
      "Unique jan_2021: ['AZ']\n",
      "Unique feb_2017: ['AZ']\n",
      "Unique feb_2018: ['AZ']\n",
      "Unique feb_2019: ['AZ']\n",
      "Unique feb_2020: ['AZ']\n",
      "Unique feb_2021: ['AZ']\n",
      "Unique jun_2017: ['AZ']\n",
      "Unique jun_2018: ['AZ']\n",
      "Unique jun_2019: ['AZ']\n",
      "Unique jun_2020: ['AZ']\n",
      "Unique jun_2021: ['AZ']\n",
      "Unique jul_2017: ['AZ']\n",
      "Unique jul_2018: ['AZ']\n",
      "Unique jul_2019: ['AZ']\n",
      "Unique jul_2020: ['AZ']\n",
      "Unique jul_2021: ['AZ']\n",
      "Unique aug_2017: ['AZ']\n",
      "Unique aug_2018: ['AZ']\n",
      "Unique aug_2019: ['AZ']\n",
      "Unique aug_2020: ['AZ']\n",
      "Unique dec_2017: ['AZ']\n",
      "Unique dec_2018: ['AZ']\n",
      "Unique dec_2019: ['AZ']\n",
      "Unique dec_2020: ['AZ']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mAZe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}