DYD_WA-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/WA-day/2017-jan-day-WA.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7e2d078e7ea21ca5d30abcd231a36f6a908ea5e5</td>\n",
       "      <td>2017-01-30 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>759</td>\n",
       "      <td>709</td>\n",
       "      <td>WA</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-01-22 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>708</td>\n",
       "      <td>687</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7e2d078e7ea21ca5d30abcd231a36f6a908ea5e5</td>\n",
       "      <td>2017-01-14 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-01-18 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>762</td>\n",
       "      <td>689</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-01-14 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171709</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-01-03 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171710</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-01-29 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171711</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-01-21 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171712</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-01-30 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171713</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-01-31 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>745</td>\n",
       "      <td>695</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>166086 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "2       7e2d078e7ea21ca5d30abcd231a36f6a908ea5e5  2017-01-30 14:50:00 UTC   \n",
       "3       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-01-22 18:00:00 UTC   \n",
       "4       7e2d078e7ea21ca5d30abcd231a36f6a908ea5e5  2017-01-14 15:15:00 UTC   \n",
       "5       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-01-18 17:25:00 UTC   \n",
       "7       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-01-14 19:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "171709  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-01-03 17:25:00 UTC   \n",
       "171710  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-01-29 17:05:00 UTC   \n",
       "171711  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-01-21 18:45:00 UTC   \n",
       "171712  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-01-30 17:40:00 UTC   \n",
       "171713  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-01-31 15:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "2          auto          hold               675                      759   \n",
       "3          auto          auto               684                      708   \n",
       "4          auto          hold               699                      755   \n",
       "5          auto          auto               684                      762   \n",
       "7          auto          hold               717                      766   \n",
       "...         ...           ...               ...                      ...   \n",
       "171709     auto          auto               710                      770   \n",
       "171710     auto          auto               711                      770   \n",
       "171711     auto          auto               712                      770   \n",
       "171712     auto          hold               702                      770   \n",
       "171713     auto          auto               682                      745   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "2                           709            WA      Ridgefield   \n",
       "3                           687            WA          Duvall   \n",
       "4                           705            WA      Ridgefield   \n",
       "5                           689            WA          Duvall   \n",
       "7                           716            WA          Duvall   \n",
       "...                         ...           ...             ...   \n",
       "171709                      710            WA  Spokane Valley   \n",
       "171710                      710            WA  Spokane Valley   \n",
       "171711                      710            WA  Spokane Valley   \n",
       "171712                      710            WA  Spokane Valley   \n",
       "171713                      695            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "2                        15             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        15             False         False            False   \n",
       "5                         5             False         False            False   \n",
       "7                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "171709                   40             False         False            False   \n",
       "171710                   40             False         False            False   \n",
       "171711                   40             False         False            False   \n",
       "171712                   40             False         False            False   \n",
       "171713                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "171709                       Gas  \n",
       "171710                       Gas  \n",
       "171711                       Gas  \n",
       "171712                       Gas  \n",
       "171713                       Gas  \n",
       "\n",
       "[166086 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>019a1d7273650d66544a79b146b515ddfa3f0531</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Vancouver</th>\n",
       "      <td>689.850000</td>\n",
       "      <td>657.500000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">02890276e16188aa7cdbc5cf916e47185c35da9a</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Lynnwood</th>\n",
       "      <td>668.666667</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>699.333333</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Lynnwood</th>\n",
       "      <td>730.694444</td>\n",
       "      <td>726.875000</td>\n",
       "      <td>726.555556</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>02f68c89b6dfd5538aec1991aad5ebc824d840e2</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Seattle</th>\n",
       "      <td>696.535714</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>698.000000</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>042f2f6da4906ec405634547472d598154b6e3fc</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>West Richland</th>\n",
       "      <td>635.772727</td>\n",
       "      <td>681.454545</td>\n",
       "      <td>635.454545</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Camas</th>\n",
       "      <td>727.700000</td>\n",
       "      <td>788.000000</td>\n",
       "      <td>686.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fe8591238eb60be06f35ae8dded14551efdfad35</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Maple Valley</th>\n",
       "      <td>670.403315</td>\n",
       "      <td>683.674033</td>\n",
       "      <td>677.359116</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">feb039fb97c44493a5351770a1e1ae13d38959a5</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Bellingham</th>\n",
       "      <td>678.333333</td>\n",
       "      <td>687.000000</td>\n",
       "      <td>678.166667</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Bellingham</th>\n",
       "      <td>683.795349</td>\n",
       "      <td>706.502326</td>\n",
       "      <td>704.293023</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Issaquah</th>\n",
       "      <td>719.666667</td>\n",
       "      <td>771.000000</td>\n",
       "      <td>709.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>543 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                          Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                  \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                689.850000   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                 668.666667   \n",
       "                                                             hold          Lynnwood                 730.694444   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                  696.535714   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland            635.772727   \n",
       "...                                                                                                        ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                    727.700000   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley             670.403315   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham               678.333333   \n",
       "                                                             hold          Bellingham               683.795349   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                 719.666667   \n",
       "\n",
       "                                                                                          TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                       657.500000   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                        780.000000   \n",
       "                                                             hold          Lynnwood                        726.875000   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                         779.000000   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland                   681.454545   \n",
       "...                                                                                                               ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                           788.000000   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley                    683.674033   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham                      687.000000   \n",
       "                                                             hold          Bellingham                      706.502326   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                        771.000000   \n",
       "\n",
       "                                                                                          TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                         \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                       640.000000   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                        699.333333   \n",
       "                                                             hold          Lynnwood                        726.555556   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                         698.000000   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland                   635.454545   \n",
       "...                                                                                                               ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                           686.000000   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley                    677.359116   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham                      678.166667   \n",
       "                                                             hold          Bellingham                      704.293023   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                        709.000000   \n",
       "\n",
       "                                                                                          Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                      5.0   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                       5.0   \n",
       "                                                             hold          Lynnwood                       5.0   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                       65.0   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland                 25.0   \n",
       "...                                                                                                       ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                          5.0   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley                  20.0   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham                    95.0   \n",
       "                                                             hold          Bellingham                    95.0   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                      10.0   \n",
       "\n",
       "                                                                                          allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                              \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                 False   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                  False   \n",
       "                                                             hold          Lynnwood                  False   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                   False   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland             False   \n",
       "...                                                                                                    ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                     False   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley              False   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham                False   \n",
       "                                                             hold          Bellingham                False   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                  False   \n",
       "\n",
       "                                                                                          Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver             False   \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood              False   \n",
       "                                                             hold          Lynnwood              False   \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle               False   \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland         False   \n",
       "...                                                                                                ...   \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                 False   \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley          False   \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham            False   \n",
       "                                                             hold          Bellingham            False   \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah              False   \n",
       "\n",
       "                                                                                          Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "019a1d7273650d66544a79b146b515ddfa3f0531 Jan   2017 heat     hold          Vancouver                 True  \n",
       "02890276e16188aa7cdbc5cf916e47185c35da9a Jan   2017 heat     auto          Lynnwood                 False  \n",
       "                                                             hold          Lynnwood                 False  \n",
       "02f68c89b6dfd5538aec1991aad5ebc824d840e2 Jan   2017 heat     auto          Seattle                  False  \n",
       "042f2f6da4906ec405634547472d598154b6e3fc Jan   2017 heat     auto          West Richland             True  \n",
       "...                                                                                                   ...  \n",
       "fcbec171f9a8c2fdd7813f1510f88d9ffb8b1789 Jan   2017 heat     auto          Camas                    False  \n",
       "fe8591238eb60be06f35ae8dded14551efdfad35 Jan   2017 heat     hold          Maple Valley             False  \n",
       "feb039fb97c44493a5351770a1e1ae13d38959a5 Jan   2017 heat     auto          Bellingham               False  \n",
       "                                                             hold          Bellingham               False  \n",
       "ff9d09ab72e971e34656b99dd7c9149bf12f7abe Jan   2017 auto     hold          Issaquah                 False  \n",
       "\n",
       "[543 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/WA/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/WA-day/2018-jan-day-WA.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32315bad8ac30472277fbfad311ff425ed4cdd5f</td>\n",
       "      <td>2018-01-19 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>629</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>160ebcafca6c48f4aef05e37f83334afe9f2282e</td>\n",
       "      <td>2018-01-16 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>775</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fife</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6b867254a7d2404c7d24b47135f9f6a41ff150c1</td>\n",
       "      <td>2018-01-19 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>795</td>\n",
       "      <td>665</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kelso</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32315bad8ac30472277fbfad311ff425ed4cdd5f</td>\n",
       "      <td>2018-01-23 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>629</td>\n",
       "      <td>610</td>\n",
       "      <td>610</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e0765853b51bfa4aed139c70f21bd2943d453261</td>\n",
       "      <td>2018-01-21 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lopez</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386718</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-01-17 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386719</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-01-17 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386720</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-01-14 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386721</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-01-17 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386722</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-01-21 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>368565 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       32315bad8ac30472277fbfad311ff425ed4cdd5f  2018-01-19 17:50:00 UTC   \n",
       "1       160ebcafca6c48f4aef05e37f83334afe9f2282e  2018-01-16 15:55:00 UTC   \n",
       "2       6b867254a7d2404c7d24b47135f9f6a41ff150c1  2018-01-19 16:15:00 UTC   \n",
       "3       32315bad8ac30472277fbfad311ff425ed4cdd5f  2018-01-23 15:55:00 UTC   \n",
       "4       e0765853b51bfa4aed139c70f21bd2943d453261  2018-01-21 18:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "386718  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-01-17 17:20:00 UTC   \n",
       "386719  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-01-17 14:35:00 UTC   \n",
       "386720  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-01-14 16:20:00 UTC   \n",
       "386721  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-01-17 14:40:00 UTC   \n",
       "386722  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-01-21 16:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               629                      630   \n",
       "1          auto          hold               708                      775   \n",
       "2          auto          hold               664                      795   \n",
       "3          heat          auto               629                      610   \n",
       "4          heat          hold               693                      691   \n",
       "...         ...           ...               ...                      ...   \n",
       "386718     auto          auto               710                      760   \n",
       "386719     heat          auto               675                      760   \n",
       "386720     heat          auto               696                      760   \n",
       "386721     heat          auto               680                      760   \n",
       "386722     auto          auto               706                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           630            WA       Rochester   \n",
       "1                           705            WA            Fife   \n",
       "2                           665            WA           Kelso   \n",
       "3                           610            WA       Rochester   \n",
       "4                           691            WA           Lopez   \n",
       "...                         ...           ...             ...   \n",
       "386718                      710            WA  Spokane Valley   \n",
       "386719                      680            WA  Spokane Valley   \n",
       "386720                      700            WA  Spokane Valley   \n",
       "386721                      680            WA  Spokane Valley   \n",
       "386722                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        45              True         False             True   \n",
       "1                        10             False         False            False   \n",
       "2                        20             False         False             True   \n",
       "3                        45              True         False             True   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "386718                   40             False         False            False   \n",
       "386719                    7             False         False            False   \n",
       "386720                    7             False         False            False   \n",
       "386721                    7             False         False            False   \n",
       "386722                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "386718                       Gas  \n",
       "386719                       Gas  \n",
       "386720                       Gas  \n",
       "386721                       Gas  \n",
       "386722                       Gas  \n",
       "\n",
       "[368565 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/WA/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/WA-day/2019-jan-day-WA.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1e05b24bec10368a01918f5049a0393b831c0ba5</td>\n",
       "      <td>2019-01-02 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>679</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>duvall</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1e05b24bec10368a01918f5049a0393b831c0ba5</td>\n",
       "      <td>2019-01-21 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>667</td>\n",
       "      <td>679</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>duvall</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a7c1d752109a7cf21b5a387be531f6a3d749bc1d</td>\n",
       "      <td>2019-01-09 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>742</td>\n",
       "      <td>702</td>\n",
       "      <td>WA</td>\n",
       "      <td>Longview</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0f3f32423e7c4bcd62bc6b6dfe86adefa0e358d7</td>\n",
       "      <td>2019-01-14 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>679</td>\n",
       "      <td>679</td>\n",
       "      <td>WA</td>\n",
       "      <td>Union</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6f6c16cbfe77d7ebf9d680277566dee999e0d4d5</td>\n",
       "      <td>2019-01-10 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>WA</td>\n",
       "      <td>Selah</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551482</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-01-01 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551483</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2019-01-09 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551484</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-01-01 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551485</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2019-01-20 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>645</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551486</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2019-01-07 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>512845 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1e05b24bec10368a01918f5049a0393b831c0ba5  2019-01-02 18:40:00 UTC   \n",
       "1       1e05b24bec10368a01918f5049a0393b831c0ba5  2019-01-21 17:15:00 UTC   \n",
       "3       a7c1d752109a7cf21b5a387be531f6a3d749bc1d  2019-01-09 18:00:00 UTC   \n",
       "4       0f3f32423e7c4bcd62bc6b6dfe86adefa0e358d7  2019-01-14 19:50:00 UTC   \n",
       "5       6f6c16cbfe77d7ebf9d680277566dee999e0d4d5  2019-01-10 17:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "551482  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-01-01 17:30:00 UTC   \n",
       "551483  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2019-01-09 13:45:00 UTC   \n",
       "551484  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-01-01 16:30:00 UTC   \n",
       "551485  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2019-01-20 15:25:00 UTC   \n",
       "551486  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2019-01-07 14:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               688                      679   \n",
       "1          heat          auto               667                      679   \n",
       "3          auto          auto               688                      742   \n",
       "4          heat          hold               674                      679   \n",
       "5          auto          hold               714                      766   \n",
       "...         ...           ...               ...                      ...   \n",
       "551482     auto          auto               702                      760   \n",
       "551483     auto          auto               687                      760   \n",
       "551484     auto          auto               704                      760   \n",
       "551485     auto          auto               645                      760   \n",
       "551486     auto          auto               682                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           660            WA          duvall   \n",
       "1                           670            WA          duvall   \n",
       "3                           702            WA        Longview   \n",
       "4                           679            WA           Union   \n",
       "5                           716            WA           Selah   \n",
       "...                         ...           ...             ...   \n",
       "551482                      710            WA  Spokane Valley   \n",
       "551483                      650            WA  Spokane Valley   \n",
       "551484                      710            WA  Spokane Valley   \n",
       "551485                      650            WA  Spokane Valley   \n",
       "551486                      650            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        30             False         False            False   \n",
       "3                         0              True         False             True   \n",
       "4                         5             False         False             True   \n",
       "5                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "551482                   40             False         False            False   \n",
       "551483                    7             False         False            False   \n",
       "551484                   40             False         False            False   \n",
       "551485                    7             False         False            False   \n",
       "551486                    7             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "551482                       Gas  \n",
       "551483                       Gas  \n",
       "551484                       Gas  \n",
       "551485                       Gas  \n",
       "551486                       Gas  \n",
       "\n",
       "[512845 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/WA/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/WA-day/2020-jan-day-WA.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17f8b089fac2e973d182b4ef01ddbff574b63268</td>\n",
       "      <td>2020-01-18 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>WA</td>\n",
       "      <td>Prosser</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17f8b089fac2e973d182b4ef01ddbff574b63268</td>\n",
       "      <td>2020-01-12 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>WA</td>\n",
       "      <td>Prosser</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6ecccbddce03e906727d6849755f10e270854a61</td>\n",
       "      <td>2020-01-04 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>658</td>\n",
       "      <td>703</td>\n",
       "      <td>653</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kalama</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4bcba90384732de8b738ef0de7e175e9b502a9be</td>\n",
       "      <td>2020-01-30 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>WA</td>\n",
       "      <td>BONNEY LAKE</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>063c5ed87f4d7bc69993dc1784c120b55768c41d</td>\n",
       "      <td>2020-01-16 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>WA</td>\n",
       "      <td>Leavenworth</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636755</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-01-05 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636756</th>\n",
       "      <td>b7ca95b07ce36a6a071763cd5db4038726c39688</td>\n",
       "      <td>2020-01-28 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636757</th>\n",
       "      <td>b7ca95b07ce36a6a071763cd5db4038726c39688</td>\n",
       "      <td>2020-01-28 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636758</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-01-12 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636759</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-01-26 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>649</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>612384 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       17f8b089fac2e973d182b4ef01ddbff574b63268  2020-01-18 18:00:00 UTC   \n",
       "2       17f8b089fac2e973d182b4ef01ddbff574b63268  2020-01-12 19:35:00 UTC   \n",
       "3       6ecccbddce03e906727d6849755f10e270854a61  2020-01-04 14:40:00 UTC   \n",
       "4       4bcba90384732de8b738ef0de7e175e9b502a9be  2020-01-30 18:30:00 UTC   \n",
       "5       063c5ed87f4d7bc69993dc1784c120b55768c41d  2020-01-16 18:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "636755  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-01-05 16:00:00 UTC   \n",
       "636756  b7ca95b07ce36a6a071763cd5db4038726c39688  2020-01-28 17:50:00 UTC   \n",
       "636757  b7ca95b07ce36a6a071763cd5db4038726c39688  2020-01-28 19:55:00 UTC   \n",
       "636758  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-01-12 18:25:00 UTC   \n",
       "636759  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-01-26 16:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               691                      765   \n",
       "2          auto          hold               691                      765   \n",
       "3          auto          auto               658                      703   \n",
       "4          heat          auto               718                      712   \n",
       "5          heat          hold               684                      709   \n",
       "...         ...           ...               ...                      ...   \n",
       "636755     heat          auto               740                      745   \n",
       "636756     heat          auto               740                      750   \n",
       "636757     heat          auto               755                      750   \n",
       "636758     heat          auto               750                      750   \n",
       "636759     heat          auto               731                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           695            WA         Prosser   \n",
       "2                           695            WA         Prosser   \n",
       "3                           653            WA          Kalama   \n",
       "4                           712            WA     BONNEY LAKE   \n",
       "5                           709            WA     Leavenworth   \n",
       "...                         ...           ...             ...   \n",
       "636755                      745            WA  Spokane Valley   \n",
       "636756                      750            WA  Spokane Valley   \n",
       "636757                      750            WA  Spokane Valley   \n",
       "636758                      750            WA  Spokane Valley   \n",
       "636759                      649            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         9             False         False             True   \n",
       "2                         9             False         False             True   \n",
       "3                        17             False         False            False   \n",
       "4                        15              True         False            False   \n",
       "5                         0              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "636755                    5             False         False            False   \n",
       "636756                   39             False         False            False   \n",
       "636757                   39             False         False            False   \n",
       "636758                    5             False         False            False   \n",
       "636759                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "636755                       Gas  \n",
       "636756                       Gas  \n",
       "636757                       Gas  \n",
       "636758                       Gas  \n",
       "636759                       Gas  \n",
       "\n",
       "[612384 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/WA/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/WA-day/2021-jan-day-WA.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c1c8018f3fb989a05665329b4dd2049f7268ffe4</td>\n",
       "      <td>2021-01-27 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>637</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>WA</td>\n",
       "      <td>Tumwater</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c1c8018f3fb989a05665329b4dd2049f7268ffe4</td>\n",
       "      <td>2021-01-27 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>630</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>WA</td>\n",
       "      <td>Tumwater</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2021-01-28 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>681</td>\n",
       "      <td>673</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2021-01-20 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>681</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>d087372e33a36ab54464198f8a69d5d7f799efae</td>\n",
       "      <td>2021-01-23 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>681</td>\n",
       "      <td>681</td>\n",
       "      <td>WA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344347</th>\n",
       "      <td>138efe444f77e07c2ee37b86d8e12df20b7ab064</td>\n",
       "      <td>2021-01-02 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>740</td>\n",
       "      <td>720</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344348</th>\n",
       "      <td>138efe444f77e07c2ee37b86d8e12df20b7ab064</td>\n",
       "      <td>2021-01-07 11:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>740</td>\n",
       "      <td>720</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344349</th>\n",
       "      <td>138efe444f77e07c2ee37b86d8e12df20b7ab064</td>\n",
       "      <td>2021-01-04 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>740</td>\n",
       "      <td>720</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344350</th>\n",
       "      <td>138efe444f77e07c2ee37b86d8e12df20b7ab064</td>\n",
       "      <td>2021-01-09 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>740</td>\n",
       "      <td>720</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344351</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-01-31 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>755</td>\n",
       "      <td>683</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>329936 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c1c8018f3fb989a05665329b4dd2049f7268ffe4  2021-01-27 19:05:00 UTC   \n",
       "2       c1c8018f3fb989a05665329b4dd2049f7268ffe4  2021-01-27 16:00:00 UTC   \n",
       "4       93949aace64e11360a850909aceeafd6d6f37900  2021-01-28 17:15:00 UTC   \n",
       "5       93949aace64e11360a850909aceeafd6d6f37900  2021-01-20 16:20:00 UTC   \n",
       "7       d087372e33a36ab54464198f8a69d5d7f799efae  2021-01-23 18:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "344347  138efe444f77e07c2ee37b86d8e12df20b7ab064  2021-01-02 12:00:00 UTC   \n",
       "344348  138efe444f77e07c2ee37b86d8e12df20b7ab064  2021-01-07 11:45:00 UTC   \n",
       "344349  138efe444f77e07c2ee37b86d8e12df20b7ab064  2021-01-04 16:05:00 UTC   \n",
       "344350  138efe444f77e07c2ee37b86d8e12df20b7ab064  2021-01-09 14:00:00 UTC   \n",
       "344351  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-01-31 17:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               637                      640   \n",
       "2          heat          hold               630                      640   \n",
       "4          heat          hold               673                      681   \n",
       "5          heat          hold               696                      681   \n",
       "7          heat          hold               670                      681   \n",
       "...         ...           ...               ...                      ...   \n",
       "344347     auto          hold               720                      740   \n",
       "344348     auto          hold               714                      740   \n",
       "344349     auto          hold               723                      740   \n",
       "344350     auto          hold               719                      740   \n",
       "344351     heat          hold               705                      755   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           640            WA        Tumwater   \n",
       "2                           640            WA        Tumwater   \n",
       "4                           673            WA       Washougal   \n",
       "5                           660            WA       Washougal   \n",
       "7                           681            WA             NaN   \n",
       "...                         ...           ...             ...   \n",
       "344347                      720            WA  Spokane Valley   \n",
       "344348                      720            WA  Spokane Valley   \n",
       "344349                      720            WA  Spokane Valley   \n",
       "344350                      720            WA  Spokane Valley   \n",
       "344351                      683            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        85             False         False             True   \n",
       "2                        85             False         False             True   \n",
       "4                        15             False         False            False   \n",
       "5                        15             False         False            False   \n",
       "7                         0              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "344347                    5             False         False             True   \n",
       "344348                    5             False         False             True   \n",
       "344349                    5             False         False             True   \n",
       "344350                    5             False         False             True   \n",
       "344351                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "2                       Electric  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "7                       Electric  \n",
       "...                          ...  \n",
       "344347                  Electric  \n",
       "344348                  Electric  \n",
       "344349                  Electric  \n",
       "344350                  Electric  \n",
       "344351                       Gas  \n",
       "\n",
       "[329936 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/WA/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>019a1d7273650d66544a79b146b515ddfa3f0531</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>689.850000</td>\n",
       "      <td>657.500000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02890276e16188aa7cdbc5cf916e47185c35da9a</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lynnwood</td>\n",
       "      <td>668.666667</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>699.333333</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02890276e16188aa7cdbc5cf916e47185c35da9a</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lynnwood</td>\n",
       "      <td>730.694444</td>\n",
       "      <td>726.875000</td>\n",
       "      <td>726.555556</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>02f68c89b6dfd5538aec1991aad5ebc824d840e2</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>696.535714</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>698.000000</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>042f2f6da4906ec405634547472d598154b6e3fc</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>West Richland</td>\n",
       "      <td>635.772727</td>\n",
       "      <td>681.454545</td>\n",
       "      <td>635.454545</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>790</th>\n",
       "      <td>fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>696.126729</td>\n",
       "      <td>698.192325</td>\n",
       "      <td>698.192325</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>fe8591238eb60be06f35ae8dded14551efdfad35</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>671.148936</td>\n",
       "      <td>700.468085</td>\n",
       "      <td>698.340426</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>792</th>\n",
       "      <td>feb039fb97c44493a5351770a1e1ae13d38959a5</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellingham</td>\n",
       "      <td>700.571429</td>\n",
       "      <td>674.214286</td>\n",
       "      <td>665.785714</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>697.588123</td>\n",
       "      <td>700.593870</td>\n",
       "      <td>700.593870</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>794</th>\n",
       "      <td>ffbba4969274f0403c05c7dca4fe68c2a9728fec</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tacoma</td>\n",
       "      <td>684.148810</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5250 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    019a1d7273650d66544a79b146b515ddfa3f0531   Jan  2017     heat   \n",
       "1    02890276e16188aa7cdbc5cf916e47185c35da9a   Jan  2017     heat   \n",
       "2    02890276e16188aa7cdbc5cf916e47185c35da9a   Jan  2017     heat   \n",
       "3    02f68c89b6dfd5538aec1991aad5ebc824d840e2   Jan  2017     heat   \n",
       "4    042f2f6da4906ec405634547472d598154b6e3fc   Jan  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "790  fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a   Jan  2021     heat   \n",
       "791  fe8591238eb60be06f35ae8dded14551efdfad35   Jan  2021     heat   \n",
       "792  feb039fb97c44493a5351770a1e1ae13d38959a5   Jan  2021     heat   \n",
       "793  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   Jan  2021     heat   \n",
       "794  ffbba4969274f0403c05c7dca4fe68c2a9728fec   Jan  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold      Vancouver            689.850000   \n",
       "1            auto       Lynnwood            668.666667   \n",
       "2            hold       Lynnwood            730.694444   \n",
       "3            auto        Seattle            696.535714   \n",
       "4            auto  West Richland            635.772727   \n",
       "..            ...            ...                   ...   \n",
       "790          hold        Colbert            696.126729   \n",
       "791          hold   Maple Valley            671.148936   \n",
       "792          hold     Bellingham            700.571429   \n",
       "793          hold         Yakima            697.588123   \n",
       "794          hold         Tacoma            684.148810   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     657.500000                   640.000000   \n",
       "1                     780.000000                   699.333333   \n",
       "2                     726.875000                   726.555556   \n",
       "3                     779.000000                   698.000000   \n",
       "4                     681.454545                   635.454545   \n",
       "..                           ...                          ...   \n",
       "790                   698.192325                   698.192325   \n",
       "791                   700.468085                   698.340426   \n",
       "792                   674.214286                   665.785714   \n",
       "793                   700.593870                   700.593870   \n",
       "794                   750.000000                   700.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False             True  \n",
       "1                    5.0             False         False            False  \n",
       "2                    5.0             False         False            False  \n",
       "3                   65.0             False         False            False  \n",
       "4                   25.0             False         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "790                 20.0             False         False            False  \n",
       "791                 20.0             False         False            False  \n",
       "792                 95.0             False         False            False  \n",
       "793                 25.0             False         False            False  \n",
       "794                 50.0              True         False             True  \n",
       "\n",
       "[5250 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/jan/\" + file)\n",
    "    WA_jan = pd.concat([WA_jan, df])\n",
    "    \n",
    "WA_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_jan.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/WA-day/2017-feb-day-WA.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-02-15 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>743</td>\n",
       "      <td>721</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-02-08 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>769</td>\n",
       "      <td>747</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>345bed8dd935c19735e0e472929079f1fc632104</td>\n",
       "      <td>2017-02-19 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Freeland</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>856bf6c249de8e8e76d12d9656f857eef51e202d</td>\n",
       "      <td>2017-02-18 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>667</td>\n",
       "      <td>708</td>\n",
       "      <td>678</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kingston</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5bb725bcc8dd210e0c3500ecea7fdbbc57a86403</td>\n",
       "      <td>2017-02-27 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>WA</td>\n",
       "      <td>issaquah</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150650</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-02-12 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150651</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-02-24 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150652</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-02-12 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150653</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-02-19 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150654</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-02-15 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>770</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>147267 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-02-15 18:20:00 UTC   \n",
       "1       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-02-08 15:35:00 UTC   \n",
       "4       345bed8dd935c19735e0e472929079f1fc632104  2017-02-19 19:20:00 UTC   \n",
       "5       856bf6c249de8e8e76d12d9656f857eef51e202d  2017-02-18 19:35:00 UTC   \n",
       "6       5bb725bcc8dd210e0c3500ecea7fdbbc57a86403  2017-02-27 13:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "150650  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-02-12 18:40:00 UTC   \n",
       "150651  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-02-24 16:20:00 UTC   \n",
       "150652  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-02-12 18:30:00 UTC   \n",
       "150653  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-02-19 16:35:00 UTC   \n",
       "150654  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-02-15 16:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               720                      743   \n",
       "1          auto          auto               725                      769   \n",
       "4          auto          hold               704                      755   \n",
       "5          heat          auto               667                      708   \n",
       "6          heat          hold               669                      671   \n",
       "...         ...           ...               ...                      ...   \n",
       "150650     auto          auto               705                      770   \n",
       "150651     auto          auto               704                      770   \n",
       "150652     auto          auto               706                      770   \n",
       "150653     auto          auto               692                      770   \n",
       "150654     auto          auto               708                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           721            WA          Duvall   \n",
       "1                           747            WA          Duvall   \n",
       "4                           705            WA        Freeland   \n",
       "5                           678            WA        Kingston   \n",
       "6                           671            WA        issaquah   \n",
       "...                         ...           ...             ...   \n",
       "150650                      710            WA  Spokane Valley   \n",
       "150651                      710            WA  Spokane Valley   \n",
       "150652                      710            WA  Spokane Valley   \n",
       "150653                      710            WA  Spokane Valley   \n",
       "150654                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "4                        25              True         False             True   \n",
       "5                         0              True         False             True   \n",
       "6                        25             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "150650                   40             False         False            False   \n",
       "150651                   40             False         False            False   \n",
       "150652                   40             False         False            False   \n",
       "150653                   40             False         False            False   \n",
       "150654                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "4                       Electric  \n",
       "5                       Electric  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "150650                       Gas  \n",
       "150651                       Gas  \n",
       "150652                       Gas  \n",
       "150653                       Gas  \n",
       "150654                       Gas  \n",
       "\n",
       "[147267 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/WA/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/WA-day/2018-feb-day-WA.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0bd80b780d9ddea1ab114b066ceebba8e12dceb8</td>\n",
       "      <td>2018-02-21 18:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>663</td>\n",
       "      <td>678</td>\n",
       "      <td>678</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lake Tapps</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e1b8814d8f007bb73d4773ce4018483dac35c75f</td>\n",
       "      <td>2018-02-25 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>WA</td>\n",
       "      <td>Wenatchee</td>\n",
       "      <td>47</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>32315bad8ac30472277fbfad311ff425ed4cdd5f</td>\n",
       "      <td>2018-02-06 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>678</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32315bad8ac30472277fbfad311ff425ed4cdd5f</td>\n",
       "      <td>2018-02-07 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>649</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32315bad8ac30472277fbfad311ff425ed4cdd5f</td>\n",
       "      <td>2018-02-12 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>651</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rochester</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365589</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-02-10 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365590</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-02-06 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365591</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-02-03 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365592</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-02-17 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365593</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-02-01 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>345965 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0bd80b780d9ddea1ab114b066ceebba8e12dceb8  2018-02-21 18:25:00 UTC   \n",
       "1       e1b8814d8f007bb73d4773ce4018483dac35c75f  2018-02-25 17:40:00 UTC   \n",
       "2       32315bad8ac30472277fbfad311ff425ed4cdd5f  2018-02-06 19:00:00 UTC   \n",
       "3       32315bad8ac30472277fbfad311ff425ed4cdd5f  2018-02-07 16:45:00 UTC   \n",
       "4       32315bad8ac30472277fbfad311ff425ed4cdd5f  2018-02-12 17:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "365589  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-02-10 16:30:00 UTC   \n",
       "365590  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-02-06 14:55:00 UTC   \n",
       "365591  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-02-03 18:35:00 UTC   \n",
       "365592  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-02-17 16:50:00 UTC   \n",
       "365593  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-02-01 16:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               663                      678   \n",
       "1          heat          hold               700                      698   \n",
       "2          heat          auto               678                      630   \n",
       "3          heat          auto               649                      630   \n",
       "4          heat          auto               651                      630   \n",
       "...         ...           ...               ...                      ...   \n",
       "365589     auto          auto               707                      760   \n",
       "365590     heat          auto               682                      760   \n",
       "365591     auto          auto               714                      760   \n",
       "365592     heat          auto               677                      760   \n",
       "365593     auto          auto               713                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           678            WA      Lake Tapps   \n",
       "1                           698            WA       Wenatchee   \n",
       "2                           630            WA       Rochester   \n",
       "3                           630            WA       Rochester   \n",
       "4                           630            WA       Rochester   \n",
       "...                         ...           ...             ...   \n",
       "365589                      710            WA  Spokane Valley   \n",
       "365590                      680            WA  Spokane Valley   \n",
       "365591                      710            WA  Spokane Valley   \n",
       "365592                      680            WA  Spokane Valley   \n",
       "365593                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        47              True         False             True   \n",
       "2                        45              True         False             True   \n",
       "3                        45              True         False             True   \n",
       "4                        45              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "365589                   40             False         False            False   \n",
       "365590                    7             False         False            False   \n",
       "365591                   40             False         False            False   \n",
       "365592                    7             False         False            False   \n",
       "365593                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "365589                       Gas  \n",
       "365590                       Gas  \n",
       "365591                       Gas  \n",
       "365592                       Gas  \n",
       "365593                       Gas  \n",
       "\n",
       "[345965 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/WA/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/WA-day/2019-feb-day-WA.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>87a69da576ca110a3e99c79e4f5aabbd789f1b72</td>\n",
       "      <td>2019-02-23 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>669</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2019-02-22 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>755</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7ae6f5212bf741042179ad1496e96ffbaa667e57</td>\n",
       "      <td>2019-02-26 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>661</td>\n",
       "      <td>717</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Suquamish</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f76a11e9855d445c2d2e1d0995444da32a92ac77</td>\n",
       "      <td>2019-02-03 17:35:00 UTC</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>WA</td>\n",
       "      <td>East Wenatchee</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>607b4876e05e51451d60f12907138e04c99faed0</td>\n",
       "      <td>2019-02-06 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>565</td>\n",
       "      <td>779</td>\n",
       "      <td>671</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384649</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-02-24 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384650</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2019-02-23 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>650</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384651</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-02-16 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384652</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-02-11 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384653</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2019-02-01 14:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>361506 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       87a69da576ca110a3e99c79e4f5aabbd789f1b72  2019-02-23 19:35:00 UTC   \n",
       "3       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2019-02-22 15:35:00 UTC   \n",
       "4       7ae6f5212bf741042179ad1496e96ffbaa667e57  2019-02-26 16:30:00 UTC   \n",
       "5       f76a11e9855d445c2d2e1d0995444da32a92ac77  2019-02-03 17:35:00 UTC   \n",
       "6       607b4876e05e51451d60f12907138e04c99faed0  2019-02-06 16:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "384649  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-02-24 17:05:00 UTC   \n",
       "384650  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2019-02-23 19:20:00 UTC   \n",
       "384651  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-02-16 16:25:00 UTC   \n",
       "384652  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-02-11 16:40:00 UTC   \n",
       "384653  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2019-02-01 14:40:00 UTC   \n",
       "\n",
       "           HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1              heat          auto               680                      669   \n",
       "3              auto          auto               687                      755   \n",
       "4              auto          auto               661                      717   \n",
       "5       auxHeatOnly          hold               703                      702   \n",
       "6              heat          auto               565                      779   \n",
       "...             ...           ...               ...                      ...   \n",
       "384649         auto          auto               695                      760   \n",
       "384650         auto          auto               650                      760   \n",
       "384651         auto          auto               706                      760   \n",
       "384652         auto          auto               699                      760   \n",
       "384653         auto          auto               690                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "1                           650            WA    Maple Valley   \n",
       "3                           690            WA          Lynden   \n",
       "4                           670            WA       Suquamish   \n",
       "5                           702            WA  East Wenatchee   \n",
       "6                           671            WA       Shoreline   \n",
       "...                         ...           ...             ...   \n",
       "384649                      710            WA  Spokane Valley   \n",
       "384650                      650            WA  Spokane Valley   \n",
       "384651                      710            WA  Spokane Valley   \n",
       "384652                      710            WA  Spokane Valley   \n",
       "384653                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         0              True         False            False   \n",
       "3                        27             False         False            False   \n",
       "4                        17             False         False             True   \n",
       "5                        15             False         False             True   \n",
       "6                        57             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "384649                   40             False         False            False   \n",
       "384650                    7             False         False            False   \n",
       "384651                   40             False         False            False   \n",
       "384652                   40             False         False            False   \n",
       "384653                    7             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                       Electric  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "384649                       Gas  \n",
       "384650                       Gas  \n",
       "384651                       Gas  \n",
       "384652                       Gas  \n",
       "384653                       Gas  \n",
       "\n",
       "[361506 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/WA/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/WA-day/2020-feb-day-WA.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5bb725bcc8dd210e0c3500ecea7fdbbc57a86403</td>\n",
       "      <td>2020-02-07 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>WA</td>\n",
       "      <td>issaquah</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d1ec16fcba08010e495c86a3f835553b6af805c8</td>\n",
       "      <td>2020-02-14 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>624</td>\n",
       "      <td>687</td>\n",
       "      <td>632</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17f8b089fac2e973d182b4ef01ddbff574b63268</td>\n",
       "      <td>2020-02-05 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>WA</td>\n",
       "      <td>Prosser</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a7c1d752109a7cf21b5a387be531f6a3d749bc1d</td>\n",
       "      <td>2020-02-11 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>757</td>\n",
       "      <td>707</td>\n",
       "      <td>WA</td>\n",
       "      <td>Longview</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c184709ee30fbd739d5d1af3d9f4e95336e33f75</td>\n",
       "      <td>2020-02-13 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>723</td>\n",
       "      <td>673</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rainier</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554429</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2020-02-23 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554430</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2020-02-15 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554431</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2020-02-09 15:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554432</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2020-02-09 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>554433</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2020-02-08 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>528702 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       5bb725bcc8dd210e0c3500ecea7fdbbc57a86403  2020-02-07 13:55:00 UTC   \n",
       "1       d1ec16fcba08010e495c86a3f835553b6af805c8  2020-02-14 15:45:00 UTC   \n",
       "2       17f8b089fac2e973d182b4ef01ddbff574b63268  2020-02-05 13:00:00 UTC   \n",
       "3       a7c1d752109a7cf21b5a387be531f6a3d749bc1d  2020-02-11 18:55:00 UTC   \n",
       "4       c184709ee30fbd739d5d1af3d9f4e95336e33f75  2020-02-13 15:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "554429  2db9bb8fd78474acbb163d049dc01c8887b2994e  2020-02-23 07:15:00 UTC   \n",
       "554430  2db9bb8fd78474acbb163d049dc01c8887b2994e  2020-02-15 16:20:00 UTC   \n",
       "554431  2db9bb8fd78474acbb163d049dc01c8887b2994e  2020-02-09 15:50:00 UTC   \n",
       "554432  2db9bb8fd78474acbb163d049dc01c8887b2994e  2020-02-09 15:25:00 UTC   \n",
       "554433  2db9bb8fd78474acbb163d049dc01c8887b2994e  2020-02-08 19:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               670                      671   \n",
       "1          heat          auto               624                      687   \n",
       "2          auto          hold               691                      765   \n",
       "3          auto          hold               704                      757   \n",
       "4          auto          hold               668                      723   \n",
       "...         ...           ...               ...                      ...   \n",
       "554429     heat          hold               739                      740   \n",
       "554430     heat          hold               738                      740   \n",
       "554431     heat          hold               748                      750   \n",
       "554432     heat          hold               746                      750   \n",
       "554433     heat          hold               758                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           671            WA        issaquah   \n",
       "1                           632            WA       Shoreline   \n",
       "2                           695            WA         Prosser   \n",
       "3                           707            WA        Longview   \n",
       "4                           673            WA         Rainier   \n",
       "...                         ...           ...             ...   \n",
       "554429                      740            WA  Spokane Valley   \n",
       "554430                      740            WA  Spokane Valley   \n",
       "554431                      750            WA  Spokane Valley   \n",
       "554432                      750            WA  Spokane Valley   \n",
       "554433                      750            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "2                         9             False         False             True   \n",
       "3                         0              True         False             True   \n",
       "4                        50             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "554429                   50              True         False            False   \n",
       "554430                   50              True         False            False   \n",
       "554431                   50              True         False            False   \n",
       "554432                   50              True         False            False   \n",
       "554433                   50              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "554429                       Gas  \n",
       "554430                       Gas  \n",
       "554431                       Gas  \n",
       "554432                       Gas  \n",
       "554433                       Gas  \n",
       "\n",
       "[528702 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/WA/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/WA-day/2021-feb-day-WA.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1324facead2e9c0d0dc8961bd772bbde65007078</td>\n",
       "      <td>2021-02-08 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>669</td>\n",
       "      <td>669</td>\n",
       "      <td>WA</td>\n",
       "      <td>Ellensburg</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2021-02-11 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>693</td>\n",
       "      <td>693</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d087372e33a36ab54464198f8a69d5d7f799efae</td>\n",
       "      <td>2021-02-03 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>WA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>665aab2d94b6161ef9c4ff9f973d7453c83f3946</td>\n",
       "      <td>2021-02-21 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>775</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Sumner</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2021-02-09 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>693</td>\n",
       "      <td>693</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322549</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-02-05 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322550</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-02-01 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322551</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-02-09 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322552</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-02-07 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322553</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2021-02-05 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>311362 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1324facead2e9c0d0dc8961bd772bbde65007078  2021-02-08 16:05:00 UTC   \n",
       "1       93949aace64e11360a850909aceeafd6d6f37900  2021-02-11 13:40:00 UTC   \n",
       "2       d087372e33a36ab54464198f8a69d5d7f799efae  2021-02-03 16:20:00 UTC   \n",
       "3       665aab2d94b6161ef9c4ff9f973d7453c83f3946  2021-02-21 16:40:00 UTC   \n",
       "4       93949aace64e11360a850909aceeafd6d6f37900  2021-02-09 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "322549  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-02-05 19:30:00 UTC   \n",
       "322550  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-02-01 17:00:00 UTC   \n",
       "322551  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-02-09 15:05:00 UTC   \n",
       "322552  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-02-07 17:15:00 UTC   \n",
       "322553  9fd474dd68222973e5214b8f5f3f394633562a5e  2021-02-05 17:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               663                      669   \n",
       "1          heat          hold               689                      693   \n",
       "2          heat          hold               689                      698   \n",
       "3          auto          hold               710                      775   \n",
       "4          heat          hold               689                      693   \n",
       "...         ...           ...               ...                      ...   \n",
       "322549     heat          hold               706                      710   \n",
       "322550     heat          hold               709                      710   \n",
       "322551     heat          hold               708                      710   \n",
       "322552     heat          hold               716                      710   \n",
       "322553     heat          hold               703                      710   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           669            WA      Ellensburg   \n",
       "1                           693            WA       Washougal   \n",
       "2                           698            WA             NaN   \n",
       "3                           705            WA          Sumner   \n",
       "4                           693            WA       Washougal   \n",
       "...                         ...           ...             ...   \n",
       "322549                      710            WA  Spokane Valley   \n",
       "322550                      710            WA  Spokane Valley   \n",
       "322551                      710            WA  Spokane Valley   \n",
       "322552                      710            WA  Spokane Valley   \n",
       "322553                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "1                        15             False         False            False   \n",
       "2                         0              True         False             True   \n",
       "3                         0             False         False            False   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "322549                   10             False         False            False   \n",
       "322550                   10             False         False            False   \n",
       "322551                   10             False         False            False   \n",
       "322552                   10             False         False            False   \n",
       "322553                   10             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "322549                       Gas  \n",
       "322550                       Gas  \n",
       "322551                       Gas  \n",
       "322552                       Gas  \n",
       "322553                       Gas  \n",
       "\n",
       "[311362 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/WA/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>019a1d7273650d66544a79b146b515ddfa3f0531</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>697.875000</td>\n",
       "      <td>655.000000</td>\n",
       "      <td>655.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02890276e16188aa7cdbc5cf916e47185c35da9a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lynnwood</td>\n",
       "      <td>732.891429</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>747.481429</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02890276e16188aa7cdbc5cf916e47185c35da9a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lynnwood</td>\n",
       "      <td>725.197740</td>\n",
       "      <td>782.028249</td>\n",
       "      <td>750.853107</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>042f2f6da4906ec405634547472d598154b6e3fc</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>West Richland</td>\n",
       "      <td>668.858044</td>\n",
       "      <td>683.873817</td>\n",
       "      <td>666.561514</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>042f2f6da4906ec405634547472d598154b6e3fc</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>West Richland</td>\n",
       "      <td>644.500000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>696.304348</td>\n",
       "      <td>699.130435</td>\n",
       "      <td>699.081285</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>fe8591238eb60be06f35ae8dded14551efdfad35</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>690.552632</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>feb039fb97c44493a5351770a1e1ae13d38959a5</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellingham</td>\n",
       "      <td>706.000000</td>\n",
       "      <td>700.625000</td>\n",
       "      <td>699.541667</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>776</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>695.624579</td>\n",
       "      <td>699.202020</td>\n",
       "      <td>699.202020</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>777</th>\n",
       "      <td>ffbba4969274f0403c05c7dca4fe68c2a9728fec</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tacoma</td>\n",
       "      <td>686.637883</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5157 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    019a1d7273650d66544a79b146b515ddfa3f0531   feb  2017     heat   \n",
       "1    02890276e16188aa7cdbc5cf916e47185c35da9a   feb  2017     heat   \n",
       "2    02890276e16188aa7cdbc5cf916e47185c35da9a   feb  2017     heat   \n",
       "3    042f2f6da4906ec405634547472d598154b6e3fc   feb  2017     heat   \n",
       "4    042f2f6da4906ec405634547472d598154b6e3fc   feb  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "773  fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a   feb  2021     heat   \n",
       "774  fe8591238eb60be06f35ae8dded14551efdfad35   feb  2021     heat   \n",
       "775  feb039fb97c44493a5351770a1e1ae13d38959a5   feb  2021     heat   \n",
       "776  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   feb  2021     heat   \n",
       "777  ffbba4969274f0403c05c7dca4fe68c2a9728fec   feb  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold      Vancouver            697.875000   \n",
       "1            auto       Lynnwood            732.891429   \n",
       "2            hold       Lynnwood            725.197740   \n",
       "3            auto  West Richland            668.858044   \n",
       "4            hold  West Richland            644.500000   \n",
       "..            ...            ...                   ...   \n",
       "773          hold        Colbert            696.304348   \n",
       "774          hold   Maple Valley            690.552632   \n",
       "775          hold     Bellingham            706.000000   \n",
       "776          hold         Yakima            695.624579   \n",
       "777          hold         Tacoma            686.637883   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     655.000000                   655.000000   \n",
       "1                     780.000000                   747.481429   \n",
       "2                     782.028249                   750.853107   \n",
       "3                     683.873817                   666.561514   \n",
       "4                     680.000000                   680.000000   \n",
       "..                           ...                          ...   \n",
       "773                   699.130435                   699.081285   \n",
       "774                   700.000000                   700.000000   \n",
       "775                   700.625000                   699.541667   \n",
       "776                   699.202020                   699.202020   \n",
       "777                   750.000000                   700.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False             True  \n",
       "1                    5.0             False         False            False  \n",
       "2                    5.0             False         False            False  \n",
       "3                   25.0             False         False             True  \n",
       "4                   25.0             False         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "773                 20.0             False         False            False  \n",
       "774                 20.0             False         False            False  \n",
       "775                 95.0             False         False            False  \n",
       "776                 25.0             False         False            False  \n",
       "777                 50.0              True         False             True  \n",
       "\n",
       "[5157 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/feb/\" + file)\n",
    "    WA_feb = pd.concat([WA_feb, df])\n",
    "    \n",
    "WA_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_feb.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/WA-day/2017-jun-day-WA.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-06-21 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>717</td>\n",
       "      <td>697</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>77ef9e32b7da9b2c1bc0e90666fe7dc8f0eac3f6</td>\n",
       "      <td>2017-06-14 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>655</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>WA</td>\n",
       "      <td>Sequim</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2cb6c68d39b98d3f2869f13f76687e5d7b6fa2cc</td>\n",
       "      <td>2017-06-24 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>810</td>\n",
       "      <td>722</td>\n",
       "      <td>WA</td>\n",
       "      <td>pullman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2017-06-22 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4a7cb170f10ebad94b27de5f60114c70ffcc5f4c</td>\n",
       "      <td>2017-06-21 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>810</td>\n",
       "      <td>610</td>\n",
       "      <td>WA</td>\n",
       "      <td>Manson</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223751</th>\n",
       "      <td>b49e68560dd35b0404964a0ee57127d0d27ef396</td>\n",
       "      <td>2017-06-06 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>750</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223752</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-06-20 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223753</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-06-20 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223754</th>\n",
       "      <td>b49e68560dd35b0404964a0ee57127d0d27ef396</td>\n",
       "      <td>2017-06-06 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>750</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223755</th>\n",
       "      <td>b49e68560dd35b0404964a0ee57127d0d27ef396</td>\n",
       "      <td>2017-06-06 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>750</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>194112 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-06-21 18:05:00 UTC   \n",
       "1       77ef9e32b7da9b2c1bc0e90666fe7dc8f0eac3f6  2017-06-14 13:20:00 UTC   \n",
       "3       2cb6c68d39b98d3f2869f13f76687e5d7b6fa2cc  2017-06-24 15:10:00 UTC   \n",
       "4       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2017-06-22 18:00:00 UTC   \n",
       "5       4a7cb170f10ebad94b27de5f60114c70ffcc5f4c  2017-06-21 13:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "223751  b49e68560dd35b0404964a0ee57127d0d27ef396  2017-06-06 14:10:00 UTC   \n",
       "223752  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-06-20 16:55:00 UTC   \n",
       "223753  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-06-20 16:20:00 UTC   \n",
       "223754  b49e68560dd35b0404964a0ee57127d0d27ef396  2017-06-06 16:20:00 UTC   \n",
       "223755  b49e68560dd35b0404964a0ee57127d0d27ef396  2017-06-06 15:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               697                      717   \n",
       "1          auto          auto               655                      685   \n",
       "3          cool          auto               706                      810   \n",
       "4          auto          hold               716                      755   \n",
       "5          auto          auto               754                      810   \n",
       "...         ...           ...               ...                      ...   \n",
       "223751     cool          auto               714                      750   \n",
       "223752     auto          auto               753                      750   \n",
       "223753     auto          auto               752                      750   \n",
       "223754     cool          auto               719                      750   \n",
       "223755     cool          auto               712                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           697            WA          Duvall   \n",
       "1                           635            WA          Sequim   \n",
       "3                           722            WA         pullman   \n",
       "4                           705            WA          Lynden   \n",
       "5                           610            WA          Manson   \n",
       "...                         ...           ...             ...   \n",
       "223751                      670            WA  Spokane Valley   \n",
       "223752                      700            WA  Spokane Valley   \n",
       "223753                      700            WA  Spokane Valley   \n",
       "223754                      670            WA  Spokane Valley   \n",
       "223755                      670            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        15             False         False             True   \n",
       "3                        10             False         False            False   \n",
       "4                        27             False         False            False   \n",
       "5                         0              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "223751                   20             False         False            False   \n",
       "223752                   40             False         False            False   \n",
       "223753                   40             False         False            False   \n",
       "223754                   20             False         False            False   \n",
       "223755                   20             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "223751                       Gas  \n",
       "223752                       Gas  \n",
       "223753                       Gas  \n",
       "223754                       Gas  \n",
       "223755                       Gas  \n",
       "\n",
       "[194112 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/WA/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/WA-day/2018-jun-day-WA.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a7c1d752109a7cf21b5a387be531f6a3d749bc1d</td>\n",
       "      <td>2018-06-05 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>WA</td>\n",
       "      <td>Longview</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14f13b7870a2feafcd280f83960e081e40f12f68</td>\n",
       "      <td>2018-06-30 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>755</td>\n",
       "      <td>685</td>\n",
       "      <td>WA</td>\n",
       "      <td>Mountlake Terrace</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a7c1d752109a7cf21b5a387be531f6a3d749bc1d</td>\n",
       "      <td>2018-06-18 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>WA</td>\n",
       "      <td>Longview</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3ef8629833e2ab64c631cd2dd696e7bf6801d118</td>\n",
       "      <td>2018-06-23 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>WA</td>\n",
       "      <td>edmonds</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d74a7bff7ea9ce1b6b961560a106f920f5fc4254</td>\n",
       "      <td>2018-06-13 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lacey</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379321</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-06-02 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379322</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2018-06-29 16:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379323</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2018-06-25 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379324</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-06-10 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379325</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2018-06-25 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>328306 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a7c1d752109a7cf21b5a387be531f6a3d749bc1d  2018-06-05 16:30:00 UTC   \n",
       "1       14f13b7870a2feafcd280f83960e081e40f12f68  2018-06-30 19:40:00 UTC   \n",
       "2       a7c1d752109a7cf21b5a387be531f6a3d749bc1d  2018-06-18 19:05:00 UTC   \n",
       "3       3ef8629833e2ab64c631cd2dd696e7bf6801d118  2018-06-23 19:25:00 UTC   \n",
       "4       d74a7bff7ea9ce1b6b961560a106f920f5fc4254  2018-06-13 15:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "379321  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-06-02 15:25:00 UTC   \n",
       "379322  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2018-06-29 16:55:00 UTC   \n",
       "379323  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2018-06-25 15:20:00 UTC   \n",
       "379324  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-06-10 15:25:00 UTC   \n",
       "379325  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2018-06-25 17:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               694                      685   \n",
       "1          auto          hold               685                      755   \n",
       "2          cool          hold               680                      675   \n",
       "3          auto          hold               730                      765   \n",
       "4          cool          hold               693                      699   \n",
       "...         ...           ...               ...                      ...   \n",
       "379321     auto          auto               714                      760   \n",
       "379322     cool          hold               737                      760   \n",
       "379323     cool          hold               734                      760   \n",
       "379324     auto          auto               713                      760   \n",
       "379325     cool          hold               743                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                           685            WA           Longview   \n",
       "1                           685            WA  Mountlake Terrace   \n",
       "2                           675            WA           Longview   \n",
       "3                           695            WA            edmonds   \n",
       "4                           699            WA              Lacey   \n",
       "...                         ...           ...                ...   \n",
       "379321                      710            WA     Spokane Valley   \n",
       "379322                      760            WA     Spokane Valley   \n",
       "379323                      760            WA     Spokane Valley   \n",
       "379324                      710            WA     Spokane Valley   \n",
       "379325                      760            WA     Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False             True   \n",
       "1                        30             False         False            False   \n",
       "2                         0              True         False             True   \n",
       "3                         5              True         False            False   \n",
       "4                         0             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "379321                   40             False         False            False   \n",
       "379322                   47             False         False            False   \n",
       "379323                   47             False         False            False   \n",
       "379324                   40             False         False            False   \n",
       "379325                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "379321                       Gas  \n",
       "379322                       Gas  \n",
       "379323                       Gas  \n",
       "379324                       Gas  \n",
       "379325                       Gas  \n",
       "\n",
       "[328306 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/WA/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/WA-day/2019-jun-day-WA.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6ecccbddce03e906727d6849755f10e270854a61</td>\n",
       "      <td>2019-06-21 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kalama</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6d7bc3daad173947535fc68f4321ed5fbcd8f65d</td>\n",
       "      <td>2019-06-13 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>685</td>\n",
       "      <td>625</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fife</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>59944f0a9aea4e4c5f6fa6c031c2ade3127a62d7</td>\n",
       "      <td>2019-06-23 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>830</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane valley</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31fe11ab0b12531660a2e2e7882f5b691c93eff5</td>\n",
       "      <td>2019-06-19 11:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>650</td>\n",
       "      <td>754</td>\n",
       "      <td>640</td>\n",
       "      <td>WA</td>\n",
       "      <td>Yelm</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>665aab2d94b6161ef9c4ff9f973d7453c83f3946</td>\n",
       "      <td>2019-06-15 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>743</td>\n",
       "      <td>693</td>\n",
       "      <td>WA</td>\n",
       "      <td>Sumner</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577122</th>\n",
       "      <td>11c690e515870873de6c58b49aa9bbbd473be566</td>\n",
       "      <td>2019-06-19 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577123</th>\n",
       "      <td>a3b5c1ff280d59051d495065da69a20817dffd49</td>\n",
       "      <td>2019-06-26 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>110</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577124</th>\n",
       "      <td>a3b5c1ff280d59051d495065da69a20817dffd49</td>\n",
       "      <td>2019-06-28 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>110</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577125</th>\n",
       "      <td>a3b5c1ff280d59051d495065da69a20817dffd49</td>\n",
       "      <td>2019-06-28 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>110</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577126</th>\n",
       "      <td>bf23519ea9e436ab70fde0e0dc22417392259a28</td>\n",
       "      <td>2019-06-05 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>538241 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6ecccbddce03e906727d6849755f10e270854a61  2019-06-21 17:55:00 UTC   \n",
       "1       6d7bc3daad173947535fc68f4321ed5fbcd8f65d  2019-06-13 14:55:00 UTC   \n",
       "2       59944f0a9aea4e4c5f6fa6c031c2ade3127a62d7  2019-06-23 16:30:00 UTC   \n",
       "3       31fe11ab0b12531660a2e2e7882f5b691c93eff5  2019-06-19 11:05:00 UTC   \n",
       "5       665aab2d94b6161ef9c4ff9f973d7453c83f3946  2019-06-15 15:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "577122  11c690e515870873de6c58b49aa9bbbd473be566  2019-06-19 17:50:00 UTC   \n",
       "577123  a3b5c1ff280d59051d495065da69a20817dffd49  2019-06-26 18:10:00 UTC   \n",
       "577124  a3b5c1ff280d59051d495065da69a20817dffd49  2019-06-28 14:40:00 UTC   \n",
       "577125  a3b5c1ff280d59051d495065da69a20817dffd49  2019-06-28 15:55:00 UTC   \n",
       "577126  bf23519ea9e436ab70fde0e0dc22417392259a28  2019-06-05 14:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               669                      685   \n",
       "1          auto          hold               680                      685   \n",
       "2          auto          hold               718                      830   \n",
       "3          heat          auto               650                      754   \n",
       "5          auto          hold               708                      743   \n",
       "...         ...           ...               ...                      ...   \n",
       "577122     heat          hold               690                      670   \n",
       "577123     heat          hold               686                      670   \n",
       "577124     heat          hold               705                      670   \n",
       "577125     heat          hold               703                      670   \n",
       "577126     heat          hold               682                      680   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           635            WA          Kalama   \n",
       "1                           625            WA            Fife   \n",
       "2                           700            WA  Spokane valley   \n",
       "3                           640            WA            Yelm   \n",
       "5                           693            WA          Sumner   \n",
       "...                         ...           ...             ...   \n",
       "577122                      670            WA         Seattle   \n",
       "577123                      670            WA         Seattle   \n",
       "577124                      670            WA         Seattle   \n",
       "577125                      670            WA         Seattle   \n",
       "577126                      680            WA         Seattle   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        17             False         False            False   \n",
       "1                         0              True         False            False   \n",
       "2                        35             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "5                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "577122                   79              True         False            False   \n",
       "577123                  110             False         False            False   \n",
       "577124                  110             False         False            False   \n",
       "577125                  110             False         False            False   \n",
       "577126                   90             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "577122                       Gas  \n",
       "577123                       Gas  \n",
       "577124                       Gas  \n",
       "577125                       Gas  \n",
       "577126                       Gas  \n",
       "\n",
       "[538241 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/WA/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/WA-day/2020-jun-day-WA.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6b0b743715b567761bebdff3bed54e65568bff4e</td>\n",
       "      <td>2020-06-25 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>652</td>\n",
       "      <td>WA</td>\n",
       "      <td>North Lynnwood</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>607b4876e05e51451d60f12907138e04c99faed0</td>\n",
       "      <td>2020-06-28 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6f6c16cbfe77d7ebf9d680277566dee999e0d4d5</td>\n",
       "      <td>2020-06-30 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>716</td>\n",
       "      <td>666</td>\n",
       "      <td>WA</td>\n",
       "      <td>Selah</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2020-06-08 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>755</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50fb39f63dbf408f1f79357a2ae7660742cc2f01</td>\n",
       "      <td>2020-06-19 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>733</td>\n",
       "      <td>683</td>\n",
       "      <td>WA</td>\n",
       "      <td>Klahanie</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558965</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-06-26 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558966</th>\n",
       "      <td>b7ca95b07ce36a6a071763cd5db4038726c39688</td>\n",
       "      <td>2020-06-12 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558967</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-06-25 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558968</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-06-05 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558969</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-06-06 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>517576 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6b0b743715b567761bebdff3bed54e65568bff4e  2020-06-25 19:50:00 UTC   \n",
       "1       607b4876e05e51451d60f12907138e04c99faed0  2020-06-28 16:00:00 UTC   \n",
       "2       6f6c16cbfe77d7ebf9d680277566dee999e0d4d5  2020-06-30 16:25:00 UTC   \n",
       "3       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2020-06-08 18:15:00 UTC   \n",
       "4       50fb39f63dbf408f1f79357a2ae7660742cc2f01  2020-06-19 18:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "558965  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-06-26 19:15:00 UTC   \n",
       "558966  b7ca95b07ce36a6a071763cd5db4038726c39688  2020-06-12 15:55:00 UTC   \n",
       "558967  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-06-25 19:05:00 UTC   \n",
       "558968  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-06-05 15:25:00 UTC   \n",
       "558969  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-06-06 15:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               712                      712   \n",
       "1          heat          hold               694                      689   \n",
       "2          auto          hold               719                      716   \n",
       "3          auto          auto               710                      755   \n",
       "4          auto          hold               733                      733   \n",
       "...         ...           ...               ...                      ...   \n",
       "558965     cool          hold               769                      760   \n",
       "558966     auto          auto               737                      760   \n",
       "558967     cool          auto               753                      760   \n",
       "558968     cool          hold               687                      760   \n",
       "558969     cool          auto               717                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           652            WA  North Lynnwood   \n",
       "1                           689            WA       Shoreline   \n",
       "2                           666            WA           Selah   \n",
       "3                           690            WA          Lynden   \n",
       "4                           683            WA        Klahanie   \n",
       "...                         ...           ...             ...   \n",
       "558965                      760            WA  Spokane Valley   \n",
       "558966                      650            WA  Spokane Valley   \n",
       "558967                      760            WA  Spokane Valley   \n",
       "558968                      760            WA  Spokane Valley   \n",
       "558969                      760            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20              True         False            False   \n",
       "1                        57             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                        27             False         False            False   \n",
       "4                        30              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "558965                   47             False         False            False   \n",
       "558966                   39             False         False            False   \n",
       "558967                   47             False         False            False   \n",
       "558968                   47             False         False            False   \n",
       "558969                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "558965                       Gas  \n",
       "558966                       Gas  \n",
       "558967                       Gas  \n",
       "558968                       Gas  \n",
       "558969                       Gas  \n",
       "\n",
       "[517576 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/WA/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/WA-day/2021-jun-day-WA.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2aa503669d32eeb4feb50b35883503ba279ce313</td>\n",
       "      <td>2021-06-29 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>766</td>\n",
       "      <td>696</td>\n",
       "      <td>WA</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>607b4876e05e51451d60f12907138e04c99faed0</td>\n",
       "      <td>2021-06-15 15:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c184709ee30fbd739d5d1af3d9f4e95336e33f75</td>\n",
       "      <td>2021-06-13 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>723</td>\n",
       "      <td>643</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rainier</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d087372e33a36ab54464198f8a69d5d7f799efae</td>\n",
       "      <td>2021-06-28 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>651</td>\n",
       "      <td>651</td>\n",
       "      <td>WA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2968d640576fa151a8a4bae2c9a445efd1cd3de1</td>\n",
       "      <td>2021-06-22 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>697</td>\n",
       "      <td>647</td>\n",
       "      <td>WA</td>\n",
       "      <td>Buckley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357933</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-06-25 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357934</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-06-23 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357935</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-06-26 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357936</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-06-23 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>357937</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-06-23 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>327789 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2aa503669d32eeb4feb50b35883503ba279ce313  2021-06-29 17:00:00 UTC   \n",
       "1       607b4876e05e51451d60f12907138e04c99faed0  2021-06-15 15:50:00 UTC   \n",
       "2       c184709ee30fbd739d5d1af3d9f4e95336e33f75  2021-06-13 16:30:00 UTC   \n",
       "3       d087372e33a36ab54464198f8a69d5d7f799efae  2021-06-28 14:45:00 UTC   \n",
       "4       2968d640576fa151a8a4bae2c9a445efd1cd3de1  2021-06-22 16:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "357933  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-06-25 17:15:00 UTC   \n",
       "357934  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-06-23 18:25:00 UTC   \n",
       "357935  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-06-26 16:25:00 UTC   \n",
       "357936  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-06-23 15:10:00 UTC   \n",
       "357937  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-06-23 16:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               756                      766   \n",
       "1          heat          hold               696                      698   \n",
       "2          auto          hold               716                      723   \n",
       "3          cool          hold               653                      651   \n",
       "4          auto          hold               694                      697   \n",
       "...         ...           ...               ...                      ...   \n",
       "357933     auto          hold               738                      760   \n",
       "357934     auto          hold               738                      760   \n",
       "357935     auto          hold               742                      760   \n",
       "357936     auto          hold               731                      760   \n",
       "357937     auto          hold               732                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           696            WA      Greenacres   \n",
       "1                           698            WA       Shoreline   \n",
       "2                           643            WA         Rainier   \n",
       "3                           651            WA             NaN   \n",
       "4                           647            WA         Buckley   \n",
       "...                         ...           ...             ...   \n",
       "357933                      710            WA  Spokane Valley   \n",
       "357934                      710            WA  Spokane Valley   \n",
       "357935                      710            WA  Spokane Valley   \n",
       "357936                      710            WA  Spokane Valley   \n",
       "357937                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        57             False         False            False   \n",
       "2                        50             False         False             True   \n",
       "3                         0              True         False             True   \n",
       "4                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "357933                   47             False         False            False   \n",
       "357934                   47             False         False            False   \n",
       "357935                   47             False         False            False   \n",
       "357936                   47             False         False            False   \n",
       "357937                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "357933                       Gas  \n",
       "357934                       Gas  \n",
       "357935                       Gas  \n",
       "357936                       Gas  \n",
       "357937                       Gas  \n",
       "\n",
       "[327789 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/WA/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02f68c89b6dfd5538aec1991aad5ebc824d840e2</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>682.755245</td>\n",
       "      <td>779.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>65.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>035f0c14a560e0656ad645cfa397bafc8e0b8bce</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>sammamish</td>\n",
       "      <td>693.392560</td>\n",
       "      <td>672.712382</td>\n",
       "      <td>660.719045</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>696.750000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>613.583333</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>042f2f6da4906ec405634547472d598154b6e3fc</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>West Richland</td>\n",
       "      <td>739.262791</td>\n",
       "      <td>760.976744</td>\n",
       "      <td>651.989535</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04449f62ff931a61cbb48eadd2757093f5c9d3af</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Richland</td>\n",
       "      <td>723.469428</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930</th>\n",
       "      <td>fe8591238eb60be06f35ae8dded14551efdfad35</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>677.400000</td>\n",
       "      <td>670.400000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>931</th>\n",
       "      <td>feb039fb97c44493a5351770a1e1ae13d38959a5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellingham</td>\n",
       "      <td>695.207792</td>\n",
       "      <td>650.363636</td>\n",
       "      <td>640.047619</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>932</th>\n",
       "      <td>fef8934c83e6b29016ef5b68074fc2d5a4da07f0</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kennewick</td>\n",
       "      <td>733.960382</td>\n",
       "      <td>736.644901</td>\n",
       "      <td>736.597946</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>933</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>713.561385</td>\n",
       "      <td>719.325103</td>\n",
       "      <td>719.325103</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>934</th>\n",
       "      <td>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>749.913043</td>\n",
       "      <td>769.891304</td>\n",
       "      <td>695.326087</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5281 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    02f68c89b6dfd5538aec1991aad5ebc824d840e2   jun  2017     heat   \n",
       "1    035f0c14a560e0656ad645cfa397bafc8e0b8bce   jun  2017     heat   \n",
       "2    038f6de02ac57361e4c383448667f66eff4e81f9   jun  2017     cool   \n",
       "3    042f2f6da4906ec405634547472d598154b6e3fc   jun  2017     cool   \n",
       "4    04449f62ff931a61cbb48eadd2757093f5c9d3af   jun  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "930  fe8591238eb60be06f35ae8dded14551efdfad35   jun  2021     heat   \n",
       "931  feb039fb97c44493a5351770a1e1ae13d38959a5   jun  2021     heat   \n",
       "932  fef8934c83e6b29016ef5b68074fc2d5a4da07f0   jun  2021     cool   \n",
       "933  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   jun  2021     cool   \n",
       "934  ff9d09ab72e971e34656b99dd7c9149bf12f7abe   jun  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto        Seattle            682.755245   \n",
       "1            hold      sammamish            693.392560   \n",
       "2            auto       Issaquah            696.750000   \n",
       "3            auto  West Richland            739.262791   \n",
       "4            auto       Richland            723.469428   \n",
       "..            ...            ...                   ...   \n",
       "930          hold   Maple Valley            670.000000   \n",
       "931          hold     Bellingham            695.207792   \n",
       "932          hold      Kennewick            733.960382   \n",
       "933          hold         Yakima            713.561385   \n",
       "934          hold       Issaquah            749.913043   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     779.000000                   680.000000   \n",
       "1                     672.712382                   660.719045   \n",
       "2                     710.000000                   613.583333   \n",
       "3                     760.976744                   651.989535   \n",
       "4                     730.000000                   650.000000   \n",
       "..                           ...                          ...   \n",
       "930                   677.400000                   670.400000   \n",
       "931                   650.363636                   640.047619   \n",
       "932                   736.644901                   736.597946   \n",
       "933                   719.325103                   719.325103   \n",
       "934                   769.891304                   695.326087   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   65.0             False         False            False  \n",
       "1                    5.0              True         False            False  \n",
       "2                   17.0             False         False             True  \n",
       "3                   25.0             False         False             True  \n",
       "4                   70.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "930                 20.0             False         False            False  \n",
       "931                 95.0             False         False            False  \n",
       "932                  0.0              True         False             True  \n",
       "933                 25.0             False         False            False  \n",
       "934                 10.0             False         False            False  \n",
       "\n",
       "[5281 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/jun/\" + file)\n",
    "    WA_jun = pd.concat([WA_jun, df])\n",
    "    \n",
    "WA_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_jun.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/WA-day/2017-jul-day-WA.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00fb48d1cbd52525aa51333669e22d5eae06f262</td>\n",
       "      <td>2017-07-07 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>795</td>\n",
       "      <td>790</td>\n",
       "      <td>WA</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2017-07-11 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>743</td>\n",
       "      <td>693</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-07-21 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>717</td>\n",
       "      <td>696</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-07-06 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>717</td>\n",
       "      <td>697</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2017-07-19 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>706</td>\n",
       "      <td>656</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244521</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-07-17 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>744</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244522</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-07-25 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244523</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-07-20 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244524</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-07-31 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244525</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-07-23 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>728</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>216413 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       00fb48d1cbd52525aa51333669e22d5eae06f262  2017-07-07 18:15:00 UTC   \n",
       "3       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2017-07-11 19:10:00 UTC   \n",
       "4       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-07-21 19:10:00 UTC   \n",
       "6       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-07-06 19:25:00 UTC   \n",
       "7       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2017-07-19 17:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "244521  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-07-17 16:00:00 UTC   \n",
       "244522  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-07-25 14:15:00 UTC   \n",
       "244523  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-07-20 15:45:00 UTC   \n",
       "244524  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-07-31 17:50:00 UTC   \n",
       "244525  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-07-23 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               730                      795   \n",
       "3          auto          hold               706                      743   \n",
       "4          auto          auto               701                      717   \n",
       "6          auto          auto               709                      717   \n",
       "7          auto          auto               724                      706   \n",
       "...         ...           ...               ...                      ...   \n",
       "244521     auto          auto               744                      760   \n",
       "244522     auto          auto               716                      760   \n",
       "244523     auto          auto               732                      760   \n",
       "244524     auto          auto               759                      760   \n",
       "244525     auto          auto               728                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           790            WA      Greenacres   \n",
       "3                           693            WA          Lynden   \n",
       "4                           696            WA          Duvall   \n",
       "6                           697            WA          Duvall   \n",
       "7                           656            WA          Lynden   \n",
       "...                         ...           ...             ...   \n",
       "244521                      660            WA  Spokane Valley   \n",
       "244522                      660            WA  Spokane Valley   \n",
       "244523                      660            WA  Spokane Valley   \n",
       "244524                      620            WA  Spokane Valley   \n",
       "244525                      660            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        50             False         False            False   \n",
       "3                        27             False         False            False   \n",
       "4                         5             False         False            False   \n",
       "6                         5             False         False            False   \n",
       "7                        27             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "244521                   40             False         False            False   \n",
       "244522                   40             False         False            False   \n",
       "244523                   40             False         False            False   \n",
       "244524                    5             False         False            False   \n",
       "244525                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "244521                       Gas  \n",
       "244522                       Gas  \n",
       "244523                       Gas  \n",
       "244524                       Gas  \n",
       "244525                       Gas  \n",
       "\n",
       "[216413 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/WA/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/WA-day/2018-jul-day-WA.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>06069495ddfa64fe44855441e844467e819dc742</td>\n",
       "      <td>2018-07-17 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>732</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d74a7bff7ea9ce1b6b961560a106f920f5fc4254</td>\n",
       "      <td>2018-07-14 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lacey</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a8f78becfdeafcd15e740373b6f71fe8a7e8b60d</td>\n",
       "      <td>2018-07-23 13:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>813</td>\n",
       "      <td>840</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Toppenish</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2cb6c68d39b98d3f2869f13f76687e5d7b6fa2cc</td>\n",
       "      <td>2018-07-29 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>810</td>\n",
       "      <td>722</td>\n",
       "      <td>WA</td>\n",
       "      <td>pullman</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a8f78becfdeafcd15e740373b6f71fe8a7e8b60d</td>\n",
       "      <td>2018-07-18 14:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>801</td>\n",
       "      <td>840</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Toppenish</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415607</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2018-07-31 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415608</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-07-11 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415609</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-07-03 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415610</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-07-15 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415612</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2018-07-31 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365468 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       06069495ddfa64fe44855441e844467e819dc742  2018-07-17 19:10:00 UTC   \n",
       "1       d74a7bff7ea9ce1b6b961560a106f920f5fc4254  2018-07-14 15:25:00 UTC   \n",
       "2       a8f78becfdeafcd15e740373b6f71fe8a7e8b60d  2018-07-23 13:05:00 UTC   \n",
       "3       2cb6c68d39b98d3f2869f13f76687e5d7b6fa2cc  2018-07-29 15:55:00 UTC   \n",
       "4       a8f78becfdeafcd15e740373b6f71fe8a7e8b60d  2018-07-18 14:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "415607  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2018-07-31 15:10:00 UTC   \n",
       "415608  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-07-11 15:00:00 UTC   \n",
       "415609  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-07-03 13:35:00 UTC   \n",
       "415610  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-07-15 19:45:00 UTC   \n",
       "415612  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2018-07-31 16:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               730                      732   \n",
       "1          cool          hold               690                      709   \n",
       "2          auto          auto               813                      840   \n",
       "3          cool          auto               756                      810   \n",
       "4          auto          auto               801                      840   \n",
       "...         ...           ...               ...                      ...   \n",
       "415607     cool          hold               746                      760   \n",
       "415608     auto          auto               725                      760   \n",
       "415609     cool          auto               716                      760   \n",
       "415610     cool          auto               736                      760   \n",
       "415612     cool          hold               752                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           670            WA    Maple Valley   \n",
       "1                           709            WA           Lacey   \n",
       "2                           680            WA       Toppenish   \n",
       "3                           722            WA         pullman   \n",
       "4                           680            WA       Toppenish   \n",
       "...                         ...           ...             ...   \n",
       "415607                      760            WA  Spokane Valley   \n",
       "415608                      680            WA  Spokane Valley   \n",
       "415609                      650            WA  Spokane Valley   \n",
       "415610                      650            WA  Spokane Valley   \n",
       "415612                      760            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        15              True         False            False   \n",
       "1                         0             False         False             True   \n",
       "2                        17              True         False             True   \n",
       "3                        10             False         False            False   \n",
       "4                        17              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "415607                   47             False         False            False   \n",
       "415608                   40             False         False            False   \n",
       "415609                    7             False         False            False   \n",
       "415610                    7             False         False            False   \n",
       "415612                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "415607                       Gas  \n",
       "415608                       Gas  \n",
       "415609                       Gas  \n",
       "415610                       Gas  \n",
       "415612                       Gas  \n",
       "\n",
       "[365468 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/WA/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/WA-day/2019-jul-day-WA.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>806efb827c0eae0f5beea97ee97d02aba96920cd</td>\n",
       "      <td>2019-07-02 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>713</td>\n",
       "      <td>663</td>\n",
       "      <td>WA</td>\n",
       "      <td>Longview</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a507c5ae8fc5894d8b75b7cc0f4cdc38ee1acf0a</td>\n",
       "      <td>2019-07-08 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>753</td>\n",
       "      <td>703</td>\n",
       "      <td>WA</td>\n",
       "      <td>Steilacoom</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4a7cb170f10ebad94b27de5f60114c70ffcc5f4c</td>\n",
       "      <td>2019-07-15 09:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>WA</td>\n",
       "      <td>Manson</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a507c5ae8fc5894d8b75b7cc0f4cdc38ee1acf0a</td>\n",
       "      <td>2019-07-27 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>703</td>\n",
       "      <td>623</td>\n",
       "      <td>WA</td>\n",
       "      <td>Steilacoom</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6d7bc3daad173947535fc68f4321ed5fbcd8f65d</td>\n",
       "      <td>2019-07-16 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>685</td>\n",
       "      <td>625</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fife</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593774</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2019-07-03 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593775</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-07-23 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593776</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-07-23 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593777</th>\n",
       "      <td>b7ca95b07ce36a6a071763cd5db4038726c39688</td>\n",
       "      <td>2019-07-22 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593778</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-07-13 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>555743 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       806efb827c0eae0f5beea97ee97d02aba96920cd  2019-07-02 19:10:00 UTC   \n",
       "1       a507c5ae8fc5894d8b75b7cc0f4cdc38ee1acf0a  2019-07-08 18:35:00 UTC   \n",
       "2       4a7cb170f10ebad94b27de5f60114c70ffcc5f4c  2019-07-15 09:55:00 UTC   \n",
       "5       a507c5ae8fc5894d8b75b7cc0f4cdc38ee1acf0a  2019-07-27 15:20:00 UTC   \n",
       "6       6d7bc3daad173947535fc68f4321ed5fbcd8f65d  2019-07-16 17:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "593774  9fd474dd68222973e5214b8f5f3f394633562a5e  2019-07-03 19:30:00 UTC   \n",
       "593775  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-07-23 15:15:00 UTC   \n",
       "593776  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-07-23 13:40:00 UTC   \n",
       "593777  b7ca95b07ce36a6a071763cd5db4038726c39688  2019-07-22 19:10:00 UTC   \n",
       "593778  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-07-13 16:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               705                      713   \n",
       "1          auto          hold               730                      753   \n",
       "2          cool          hold               726                      742   \n",
       "5          auto          hold               752                      703   \n",
       "6          auto          hold               689                      685   \n",
       "...         ...           ...               ...                      ...   \n",
       "593774     auto          auto               746                      760   \n",
       "593775     cool          hold               716                      760   \n",
       "593776     cool          hold               708                      760   \n",
       "593777     cool          auto               740                      760   \n",
       "593778     cool          auto               760                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           663            WA        Longview   \n",
       "1                           703            WA      Steilacoom   \n",
       "2                           742            WA          Manson   \n",
       "5                           623            WA      Steilacoom   \n",
       "6                           625            WA            Fife   \n",
       "...                         ...           ...             ...   \n",
       "593774                      700            WA  Spokane Valley   \n",
       "593775                      760            WA  Spokane Valley   \n",
       "593776                      760            WA  Spokane Valley   \n",
       "593777                      760            WA  Spokane Valley   \n",
       "593778                      760            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10              True         False             True   \n",
       "1                        50             False         False            False   \n",
       "2                         0              True         False             True   \n",
       "5                        50             False         False            False   \n",
       "6                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "593774                   10             False         False            False   \n",
       "593775                   47             False         False            False   \n",
       "593776                   47             False         False            False   \n",
       "593777                   39             False         False            False   \n",
       "593778                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "593774                       Gas  \n",
       "593775                       Gas  \n",
       "593776                       Gas  \n",
       "593777                       Gas  \n",
       "593778                       Gas  \n",
       "\n",
       "[555743 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/WA/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/WA-day/2020-jul-day-WA.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a56e1e8f35a7803368e77beea18a9f35da2de241</td>\n",
       "      <td>2020-07-30 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>777</td>\n",
       "      <td>795</td>\n",
       "      <td>790</td>\n",
       "      <td>WA</td>\n",
       "      <td>Leavenworth</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>607b4876e05e51451d60f12907138e04c99faed0</td>\n",
       "      <td>2020-07-05 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6b867254a7d2404c7d24b47135f9f6a41ff150c1</td>\n",
       "      <td>2020-07-27 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>680</td>\n",
       "      <td>686</td>\n",
       "      <td>630</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kelso</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4a7cb170f10ebad94b27de5f60114c70ffcc5f4c</td>\n",
       "      <td>2020-07-23 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>WA</td>\n",
       "      <td>Manson</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25d90ac98d87776e64bb6d04d82909f97dd150ea</td>\n",
       "      <td>2020-07-27 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>774</td>\n",
       "      <td>774</td>\n",
       "      <td>WA</td>\n",
       "      <td>Moses Lake</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572536</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-07-07 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572537</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-07-08 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572538</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-07-08 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572539</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-07-08 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572540</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-07-30 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>770</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>507390 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a56e1e8f35a7803368e77beea18a9f35da2de241  2020-07-30 15:05:00 UTC   \n",
       "1       607b4876e05e51451d60f12907138e04c99faed0  2020-07-05 15:35:00 UTC   \n",
       "2       6b867254a7d2404c7d24b47135f9f6a41ff150c1  2020-07-27 15:20:00 UTC   \n",
       "3       4a7cb170f10ebad94b27de5f60114c70ffcc5f4c  2020-07-23 15:20:00 UTC   \n",
       "4       25d90ac98d87776e64bb6d04d82909f97dd150ea  2020-07-27 13:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "572536  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-07-07 16:20:00 UTC   \n",
       "572537  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-07-08 15:45:00 UTC   \n",
       "572538  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-07-08 14:00:00 UTC   \n",
       "572539  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-07-08 17:15:00 UTC   \n",
       "572540  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-07-30 14:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               777                      795   \n",
       "1          heat          hold               687                      689   \n",
       "2          auto          auto               680                      686   \n",
       "3          cool          hold               754                      765   \n",
       "4          cool          hold               723                      774   \n",
       "...         ...           ...               ...                      ...   \n",
       "572536     cool          auto               716                      760   \n",
       "572537     cool          auto               707                      760   \n",
       "572538     cool          auto               700                      760   \n",
       "572539     cool          auto               709                      760   \n",
       "572540     cool          auto               688                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           790            WA     Leavenworth   \n",
       "1                           689            WA       Shoreline   \n",
       "2                           630            WA           Kelso   \n",
       "3                           765            WA          Manson   \n",
       "4                           774            WA      Moses Lake   \n",
       "...                         ...           ...             ...   \n",
       "572536                      760            WA  Spokane Valley   \n",
       "572537                      760            WA  Spokane Valley   \n",
       "572538                      760            WA  Spokane Valley   \n",
       "572539                      760            WA  Spokane Valley   \n",
       "572540                      770            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20              True         False             True   \n",
       "1                        57             False         False            False   \n",
       "2                        20             False         False             True   \n",
       "3                         0              True         False             True   \n",
       "4                         5             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "572536                   47             False         False            False   \n",
       "572537                   47             False         False            False   \n",
       "572538                   47             False         False            False   \n",
       "572539                   47             False         False            False   \n",
       "572540                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "572536                       Gas  \n",
       "572537                       Gas  \n",
       "572538                       Gas  \n",
       "572539                       Gas  \n",
       "572540                       Gas  \n",
       "\n",
       "[507390 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/WA/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/WA-day/2021-jul-day-WA.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7727cb56d25bb11ff9adbd0e2f0528f5f6767aed</td>\n",
       "      <td>2021-07-05 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>733</td>\n",
       "      <td>733</td>\n",
       "      <td>WA</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>360d78e4c7af7940b12c7d6c1f46b528ff50582f</td>\n",
       "      <td>2021-07-17 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>815</td>\n",
       "      <td>765</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>18</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>856bf6c249de8e8e76d12d9656f857eef51e202d</td>\n",
       "      <td>2021-07-04 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kingston</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7727cb56d25bb11ff9adbd0e2f0528f5f6767aed</td>\n",
       "      <td>2021-07-03 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>733</td>\n",
       "      <td>733</td>\n",
       "      <td>WA</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6f6c16cbfe77d7ebf9d680277566dee999e0d4d5</td>\n",
       "      <td>2021-07-14 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>727</td>\n",
       "      <td>677</td>\n",
       "      <td>WA</td>\n",
       "      <td>Selah</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289991</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-07-23 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290001</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-07-14 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290002</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-07-14 15:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290003</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-07-14 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290004</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2021-07-14 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259404 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7727cb56d25bb11ff9adbd0e2f0528f5f6767aed  2021-07-05 18:25:00 UTC   \n",
       "3       360d78e4c7af7940b12c7d6c1f46b528ff50582f  2021-07-17 19:45:00 UTC   \n",
       "4       856bf6c249de8e8e76d12d9656f857eef51e202d  2021-07-04 16:40:00 UTC   \n",
       "5       7727cb56d25bb11ff9adbd0e2f0528f5f6767aed  2021-07-03 19:00:00 UTC   \n",
       "6       6f6c16cbfe77d7ebf9d680277566dee999e0d4d5  2021-07-14 15:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "289991  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-07-23 19:45:00 UTC   \n",
       "290001  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-07-14 14:10:00 UTC   \n",
       "290002  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-07-14 15:50:00 UTC   \n",
       "290003  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-07-14 16:25:00 UTC   \n",
       "290004  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2021-07-14 16:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               721                      733   \n",
       "3          auto          hold               767                      815   \n",
       "4          cool          hold               712                      757   \n",
       "5          cool          hold               733                      733   \n",
       "6          auto          hold               727                      727   \n",
       "...         ...           ...               ...                      ...   \n",
       "289991     cool          hold               739                      750   \n",
       "290001     cool          hold               709                      760   \n",
       "290002     cool          hold               715                      760   \n",
       "290003     cool          hold               723                      760   \n",
       "290004     cool          hold               726                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           733            WA    Maple Valley   \n",
       "3                           765            WA          Duvall   \n",
       "4                           757            WA        Kingston   \n",
       "5                           733            WA    Maple Valley   \n",
       "6                           677            WA           Selah   \n",
       "...                         ...           ...             ...   \n",
       "289991                      750            WA  Spokane Valley   \n",
       "290001                      760            WA  Spokane Valley   \n",
       "290002                      760            WA  Spokane Valley   \n",
       "290003                      760            WA  Spokane Valley   \n",
       "290004                      760            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "3                        18             False         False            False   \n",
       "4                         0              True         False             True   \n",
       "5                         0              True         False            False   \n",
       "6                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "289991                   47             False         False            False   \n",
       "290001                   47             False         False            False   \n",
       "290002                   47             False         False            False   \n",
       "290003                   47             False         False            False   \n",
       "290004                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "289991                       Gas  \n",
       "290001                       Gas  \n",
       "290002                       Gas  \n",
       "290003                       Gas  \n",
       "290004                       Gas  \n",
       "\n",
       "[259404 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/WA/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00fb48d1cbd52525aa51333669e22d5eae06f262</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>727.366972</td>\n",
       "      <td>792.183486</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Richland</td>\n",
       "      <td>720.714286</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>700.208333</td>\n",
       "      <td>700.416667</td>\n",
       "      <td>698.333333</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>042f2f6da4906ec405634547472d598154b6e3fc</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>West Richland</td>\n",
       "      <td>738.677596</td>\n",
       "      <td>739.677596</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04449f62ff931a61cbb48eadd2757093f5c9d3af</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Richland</td>\n",
       "      <td>746.846154</td>\n",
       "      <td>749.249417</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>fca85463bc93d6be856df0c32a4bce76a09d27b6</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bothell</td>\n",
       "      <td>695.333333</td>\n",
       "      <td>698.000000</td>\n",
       "      <td>700.666667</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>724.085470</td>\n",
       "      <td>731.346154</td>\n",
       "      <td>660.711538</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>fef8934c83e6b29016ef5b68074fc2d5a4da07f0</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kennewick</td>\n",
       "      <td>750.236480</td>\n",
       "      <td>757.266728</td>\n",
       "      <td>757.266728</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>719.565780</td>\n",
       "      <td>716.414634</td>\n",
       "      <td>716.414634</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>743.166667</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4473 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00fb48d1cbd52525aa51333669e22d5eae06f262   jul  2017     cool   \n",
       "1    02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb   jul  2017     cool   \n",
       "2    038f6de02ac57361e4c383448667f66eff4e81f9   jul  2017     cool   \n",
       "3    042f2f6da4906ec405634547472d598154b6e3fc   jul  2017     cool   \n",
       "4    04449f62ff931a61cbb48eadd2757093f5c9d3af   jul  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "560  fca85463bc93d6be856df0c32a4bce76a09d27b6   jul  2021     heat   \n",
       "561  fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a   jul  2021     auto   \n",
       "562  fef8934c83e6b29016ef5b68074fc2d5a4da07f0   jul  2021     cool   \n",
       "563  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   jul  2021     cool   \n",
       "564  ff9d09ab72e971e34656b99dd7c9149bf12f7abe   jul  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold     Greenacres            727.366972   \n",
       "1            hold       Richland            720.714286   \n",
       "2            hold       Issaquah            700.208333   \n",
       "3            auto  West Richland            738.677596   \n",
       "4            hold       Richland            746.846154   \n",
       "..            ...            ...                   ...   \n",
       "560          hold        Bothell            695.333333   \n",
       "561          hold        Colbert            724.085470   \n",
       "562          hold      Kennewick            750.236480   \n",
       "563          hold         Yakima            719.565780   \n",
       "564          hold       Issaquah            743.166667   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     792.183486                   790.000000   \n",
       "1                     720.000000                   720.000000   \n",
       "2                     700.416667                   698.333333   \n",
       "3                     739.677596                   650.000000   \n",
       "4                     749.249417                   650.000000   \n",
       "..                           ...                          ...   \n",
       "560                   698.000000                   700.666667   \n",
       "561                   731.346154                   660.711538   \n",
       "562                   757.266728                   757.266728   \n",
       "563                   716.414634                   716.414634   \n",
       "564                   740.000000                   700.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   50.0             False         False            False  \n",
       "1                   70.0              True         False             True  \n",
       "2                   17.0             False         False             True  \n",
       "3                   25.0             False         False             True  \n",
       "4                   70.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "560                 25.0             False         False            False  \n",
       "561                 20.0             False         False            False  \n",
       "562                  0.0              True         False             True  \n",
       "563                 25.0             False         False            False  \n",
       "564                 10.0             False         False            False  \n",
       "\n",
       "[4473 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/jul/\" + file)\n",
    "    WA_jul = pd.concat([WA_jul, df])\n",
    "    \n",
    "WA_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_jul.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/WA-day/2017-aug-day-WA.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78abf2ea6bdcf569be6e0675588ad3d5b1aa8cde</td>\n",
       "      <td>2017-08-13 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>764</td>\n",
       "      <td>694</td>\n",
       "      <td>WA</td>\n",
       "      <td>Graham</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19f5ae392e4335e5b28f5e2762003325c04dd3e9</td>\n",
       "      <td>2017-08-19 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>685</td>\n",
       "      <td>635</td>\n",
       "      <td>WA</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5bb725bcc8dd210e0c3500ecea7fdbbc57a86403</td>\n",
       "      <td>2017-08-06 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>779</td>\n",
       "      <td>644</td>\n",
       "      <td>WA</td>\n",
       "      <td>issaquah</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5bb725bcc8dd210e0c3500ecea7fdbbc57a86403</td>\n",
       "      <td>2017-08-06 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>779</td>\n",
       "      <td>644</td>\n",
       "      <td>WA</td>\n",
       "      <td>issaquah</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2017-08-18 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>717</td>\n",
       "      <td>697</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253296</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-08-18 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253297</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-08-20 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253298</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-08-09 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253299</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-08-19 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253300</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2017-08-19 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>217585 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       78abf2ea6bdcf569be6e0675588ad3d5b1aa8cde  2017-08-13 18:30:00 UTC   \n",
       "3       19f5ae392e4335e5b28f5e2762003325c04dd3e9  2017-08-19 13:45:00 UTC   \n",
       "4       5bb725bcc8dd210e0c3500ecea7fdbbc57a86403  2017-08-06 17:25:00 UTC   \n",
       "6       5bb725bcc8dd210e0c3500ecea7fdbbc57a86403  2017-08-06 14:55:00 UTC   \n",
       "7       36318fcb544174abebcffdbd2e45415c9ac165fc  2017-08-18 19:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "253296  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-08-18 15:45:00 UTC   \n",
       "253297  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-08-20 18:35:00 UTC   \n",
       "253298  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-08-09 15:10:00 UTC   \n",
       "253299  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-08-19 14:35:00 UTC   \n",
       "253300  c0522086d24320d88a6de0cfc363c999e6a7d61e  2017-08-19 14:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               732                      764   \n",
       "3          auto          auto               684                      685   \n",
       "4          heat          hold               735                      779   \n",
       "6          heat          hold               720                      779   \n",
       "7          auto          auto               706                      717   \n",
       "...         ...           ...               ...                      ...   \n",
       "253296     cool          auto               725                      760   \n",
       "253297     auto          auto               723                      760   \n",
       "253298     auto          auto               732                      760   \n",
       "253299     cool          auto               731                      760   \n",
       "253300     cool          auto               730                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           694            WA          Graham   \n",
       "3                           635            WA         Colbert   \n",
       "4                           644            WA        issaquah   \n",
       "6                           644            WA        issaquah   \n",
       "7                           697            WA          Duvall   \n",
       "...                         ...           ...             ...   \n",
       "253296                      620            WA  Spokane Valley   \n",
       "253297                      620            WA  Spokane Valley   \n",
       "253298                      620            WA  Spokane Valley   \n",
       "253299                      620            WA  Spokane Valley   \n",
       "253300                      620            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False             True   \n",
       "3                         0             False         False            False   \n",
       "4                        25             False         False            False   \n",
       "6                        25             False         False            False   \n",
       "7                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "253296                    5             False         False            False   \n",
       "253297                    5             False         False            False   \n",
       "253298                    5             False         False            False   \n",
       "253299                    5             False         False            False   \n",
       "253300                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "253296                       Gas  \n",
       "253297                       Gas  \n",
       "253298                       Gas  \n",
       "253299                       Gas  \n",
       "253300                       Gas  \n",
       "\n",
       "[217585 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/WA/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/WA-day/2018-aug-day-WA.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>135fe8e4c9cced46d18754ccce1efee55f08667a</td>\n",
       "      <td>2018-08-20 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>755</td>\n",
       "      <td>715</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>135fe8e4c9cced46d18754ccce1efee55f08667a</td>\n",
       "      <td>2018-08-25 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>748</td>\n",
       "      <td>708</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6f905dab669493b777218abec1f282384fba57f7</td>\n",
       "      <td>2018-08-25 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>650</td>\n",
       "      <td>840</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Buckley</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1389d87081f3336bea6e1f3f37a8d0fa465773d7</td>\n",
       "      <td>2018-08-15 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>696</td>\n",
       "      <td>646</td>\n",
       "      <td>WA</td>\n",
       "      <td>Kelso</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>135fe8e4c9cced46d18754ccce1efee55f08667a</td>\n",
       "      <td>2018-08-31 15:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>738</td>\n",
       "      <td>718</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434135</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-08-05 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434136</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-08-18 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434137</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-08-13 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434138</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-08-29 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434139</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-08-11 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>379078 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       135fe8e4c9cced46d18754ccce1efee55f08667a  2018-08-20 16:55:00 UTC   \n",
       "4       135fe8e4c9cced46d18754ccce1efee55f08667a  2018-08-25 16:15:00 UTC   \n",
       "5       6f905dab669493b777218abec1f282384fba57f7  2018-08-25 16:55:00 UTC   \n",
       "6       1389d87081f3336bea6e1f3f37a8d0fa465773d7  2018-08-15 17:20:00 UTC   \n",
       "9       135fe8e4c9cced46d18754ccce1efee55f08667a  2018-08-31 15:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "434135  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-08-05 15:30:00 UTC   \n",
       "434136  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-08-18 15:00:00 UTC   \n",
       "434137  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-08-13 13:40:00 UTC   \n",
       "434138  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-08-29 12:55:00 UTC   \n",
       "434139  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-08-11 18:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          auto               719                      755   \n",
       "4          auto          auto               701                      748   \n",
       "5          auto          hold               650                      840   \n",
       "6          auto          hold               670                      696   \n",
       "9          auto          auto               709                      738   \n",
       "...         ...           ...               ...                      ...   \n",
       "434135     cool          auto               748                      760   \n",
       "434136     cool          auto               750                      760   \n",
       "434137     cool          auto               737                      760   \n",
       "434138     cool          auto               693                      760   \n",
       "434139     cool          auto               763                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "1                           715            WA         Shelton   \n",
       "4                           708            WA         Shelton   \n",
       "5                           650            WA         Buckley   \n",
       "6                           646            WA           Kelso   \n",
       "9                           718            WA         Shelton   \n",
       "...                         ...           ...             ...   \n",
       "434135                      650            WA  Spokane Valley   \n",
       "434136                      650            WA  Spokane Valley   \n",
       "434137                      650            WA  Spokane Valley   \n",
       "434138                      650            WA  Spokane Valley   \n",
       "434139                      650            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        20              True         False             True   \n",
       "4                        20              True         False             True   \n",
       "5                        20             False         False            False   \n",
       "6                         0              True         False             True   \n",
       "9                        20              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "434135                    7             False         False            False   \n",
       "434136                    7             False         False            False   \n",
       "434137                    7             False         False            False   \n",
       "434138                    7             False         False            False   \n",
       "434139                    7             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                       Electric  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "6                       Electric  \n",
       "9                       Electric  \n",
       "...                          ...  \n",
       "434135                       Gas  \n",
       "434136                       Gas  \n",
       "434137                       Gas  \n",
       "434138                       Gas  \n",
       "434139                       Gas  \n",
       "\n",
       "[379078 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/WA/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/WA-day/2019-aug-day-WA.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2234a1cc57e3f3272badb2d68217b38881bd0af7</td>\n",
       "      <td>2019-08-08 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>733</td>\n",
       "      <td>733</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fircrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c184709ee30fbd739d5d1af3d9f4e95336e33f75</td>\n",
       "      <td>2019-08-31 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>698</td>\n",
       "      <td>618</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rainier</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>36318fcb544174abebcffdbd2e45415c9ac165fc</td>\n",
       "      <td>2019-08-15 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>698</td>\n",
       "      <td>678</td>\n",
       "      <td>WA</td>\n",
       "      <td>Duvall</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>411c8de7ca48740e170056d9f091ee69d2ab5456</td>\n",
       "      <td>2019-08-29 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>WA</td>\n",
       "      <td>University Place</td>\n",
       "      <td>59</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>d05718c39dcc73e214e29957ade540484e37aedb</td>\n",
       "      <td>2019-08-11 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>732</td>\n",
       "      <td>702</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shelton</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580672</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-08-09 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580673</th>\n",
       "      <td>9fd474dd68222973e5214b8f5f3f394633562a5e</td>\n",
       "      <td>2019-08-10 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580674</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2019-08-15 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580675</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-08-14 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>580676</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2019-08-14 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>535750 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2234a1cc57e3f3272badb2d68217b38881bd0af7  2019-08-08 19:30:00 UTC   \n",
       "3       c184709ee30fbd739d5d1af3d9f4e95336e33f75  2019-08-31 13:30:00 UTC   \n",
       "5       36318fcb544174abebcffdbd2e45415c9ac165fc  2019-08-15 19:15:00 UTC   \n",
       "6       411c8de7ca48740e170056d9f091ee69d2ab5456  2019-08-29 14:25:00 UTC   \n",
       "7       d05718c39dcc73e214e29957ade540484e37aedb  2019-08-11 18:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "580672  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-08-09 15:25:00 UTC   \n",
       "580673  9fd474dd68222973e5214b8f5f3f394633562a5e  2019-08-10 19:35:00 UTC   \n",
       "580674  0c934c0fae5270491f44ada0ec919994dc79c6c0  2019-08-15 16:50:00 UTC   \n",
       "580675  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-08-14 18:55:00 UTC   \n",
       "580676  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2019-08-14 16:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               730                      733   \n",
       "3          auto          hold               695                      698   \n",
       "5          auto          hold               699                      698   \n",
       "6          cool          hold               678                      675   \n",
       "7          auto          hold               697                      732   \n",
       "...         ...           ...               ...                      ...   \n",
       "580672     cool          auto               732                      760   \n",
       "580673     auto          hold               764                      760   \n",
       "580674     auto          auto               733                      760   \n",
       "580675     cool          hold               742                      760   \n",
       "580676     cool          hold               724                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                           733            WA          Fircrest   \n",
       "3                           618            WA           Rainier   \n",
       "5                           678            WA            Duvall   \n",
       "6                           675            WA  University Place   \n",
       "7                           702            WA           Shelton   \n",
       "...                         ...           ...               ...   \n",
       "580672                      760            WA    Spokane Valley   \n",
       "580673                      690            WA    Spokane Valley   \n",
       "580674                      700            WA    Spokane Valley   \n",
       "580675                      760            WA    Spokane Valley   \n",
       "580676                      760            WA    Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "3                        50             False         False             True   \n",
       "5                         5             False         False            False   \n",
       "6                        59              True         False            False   \n",
       "7                        15              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "580672                   47             False         False            False   \n",
       "580673                   10             False         False            False   \n",
       "580674                   40             False         False            False   \n",
       "580675                   47             False         False            False   \n",
       "580676                   47             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "3                       Electric  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "7                       Electric  \n",
       "...                          ...  \n",
       "580672                       Gas  \n",
       "580673                       Gas  \n",
       "580674                       Gas  \n",
       "580675                       Gas  \n",
       "580676                       Gas  \n",
       "\n",
       "[535750 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/WA/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/WA-day/2020-aug-day-WA.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>607b4876e05e51451d60f12907138e04c99faed0</td>\n",
       "      <td>2020-08-15 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>WA</td>\n",
       "      <td>Shoreline</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a56e1e8f35a7803368e77beea18a9f35da2de241</td>\n",
       "      <td>2020-08-04 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>795</td>\n",
       "      <td>790</td>\n",
       "      <td>WA</td>\n",
       "      <td>Leavenworth</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2bbc85849c4334c7efad2fea32c5dbee4c2fee3d</td>\n",
       "      <td>2020-08-30 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>658</td>\n",
       "      <td>646</td>\n",
       "      <td>WA</td>\n",
       "      <td>Dayton</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c184709ee30fbd739d5d1af3d9f4e95336e33f75</td>\n",
       "      <td>2020-08-19 12:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>704</td>\n",
       "      <td>604</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rainier</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>160ebcafca6c48f4aef05e37f83334afe9f2282e</td>\n",
       "      <td>2020-08-29 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>719</td>\n",
       "      <td>669</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fife</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557183</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-08-12 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557184</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2020-08-26 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557185</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-08-12 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>711</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557186</th>\n",
       "      <td>b5420ee9899999d0bf392fdf370f9683f8cf9ba9</td>\n",
       "      <td>2020-08-12 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557187</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2020-08-27 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>513271 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       607b4876e05e51451d60f12907138e04c99faed0  2020-08-15 16:05:00 UTC   \n",
       "2       a56e1e8f35a7803368e77beea18a9f35da2de241  2020-08-04 17:50:00 UTC   \n",
       "3       2bbc85849c4334c7efad2fea32c5dbee4c2fee3d  2020-08-30 17:15:00 UTC   \n",
       "4       c184709ee30fbd739d5d1af3d9f4e95336e33f75  2020-08-19 12:45:00 UTC   \n",
       "5       160ebcafca6c48f4aef05e37f83334afe9f2282e  2020-08-29 18:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "557183  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-08-12 17:20:00 UTC   \n",
       "557184  0c934c0fae5270491f44ada0ec919994dc79c6c0  2020-08-26 16:45:00 UTC   \n",
       "557185  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-08-12 15:35:00 UTC   \n",
       "557186  b5420ee9899999d0bf392fdf370f9683f8cf9ba9  2020-08-12 17:25:00 UTC   \n",
       "557187  0c934c0fae5270491f44ada0ec919994dc79c6c0  2020-08-27 15:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               707                      689   \n",
       "2          cool          hold               769                      795   \n",
       "3          cool          hold               707                      658   \n",
       "4          auto          hold               706                      704   \n",
       "5          auto          hold               715                      719   \n",
       "...         ...           ...               ...                      ...   \n",
       "557183     cool          auto               711                      760   \n",
       "557184     auto          auto               707                      760   \n",
       "557185     cool          auto               711                      760   \n",
       "557186     cool          auto               712                      760   \n",
       "557187     auto          auto               692                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           689            WA       Shoreline   \n",
       "2                           790            WA     Leavenworth   \n",
       "3                           646            WA          Dayton   \n",
       "4                           604            WA         Rainier   \n",
       "5                           669            WA            Fife   \n",
       "...                         ...           ...             ...   \n",
       "557183                      760            WA  Spokane Valley   \n",
       "557184                      690            WA  Spokane Valley   \n",
       "557185                      760            WA  Spokane Valley   \n",
       "557186                      760            WA  Spokane Valley   \n",
       "557187                      690            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        57             False         False            False   \n",
       "2                        20              True         False             True   \n",
       "3                        57             False         False            False   \n",
       "4                        50             False         False             True   \n",
       "5                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "557183                   47             False         False            False   \n",
       "557184                   40             False         False            False   \n",
       "557185                   47             False         False            False   \n",
       "557186                   47             False         False            False   \n",
       "557187                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "557183                       Gas  \n",
       "557184                       Gas  \n",
       "557185                       Gas  \n",
       "557186                       Gas  \n",
       "557187                       Gas  \n",
       "\n",
       "[513271 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/WA/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00fb48d1cbd52525aa51333669e22d5eae06f262</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>693.000000</td>\n",
       "      <td>657.000000</td>\n",
       "      <td>602.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>019a1d7273650d66544a79b146b515ddfa3f0531</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>701.909091</td>\n",
       "      <td>715.727273</td>\n",
       "      <td>715.727273</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Richland</td>\n",
       "      <td>728.562500</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>720.500000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>693.470588</td>\n",
       "      <td>687.294118</td>\n",
       "      <td>687.294118</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1087</th>\n",
       "      <td>fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>728.868313</td>\n",
       "      <td>735.566667</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1088</th>\n",
       "      <td>fef8934c83e6b29016ef5b68074fc2d5a4da07f0</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kennewick</td>\n",
       "      <td>724.901985</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>740.007218</td>\n",
       "      <td>763.828740</td>\n",
       "      <td>687.930446</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1090</th>\n",
       "      <td>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>762.409091</td>\n",
       "      <td>763.636364</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1091</th>\n",
       "      <td>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>765.450000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3612 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00fb48d1cbd52525aa51333669e22d5eae06f262   aug  2017     auto   \n",
       "1     019a1d7273650d66544a79b146b515ddfa3f0531   aug  2017     cool   \n",
       "2     02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb   aug  2017     cool   \n",
       "3     038f6de02ac57361e4c383448667f66eff4e81f9   aug  2017     cool   \n",
       "4     038f6de02ac57361e4c383448667f66eff4e81f9   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1087  fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a   aug  2020     auto   \n",
       "1088  fef8934c83e6b29016ef5b68074fc2d5a4da07f0   aug  2020     cool   \n",
       "1089  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   aug  2020     auto   \n",
       "1090  ff9d09ab72e971e34656b99dd7c9149bf12f7abe   aug  2020     auto   \n",
       "1091  ff9d09ab72e971e34656b99dd7c9149bf12f7abe   aug  2020     auto   \n",
       "\n",
       "     CalendarEvent        City  Temperature_ctrl_ave  \\\n",
       "0             hold  Greenacres            693.000000   \n",
       "1             hold   Vancouver            701.909091   \n",
       "2             auto    Richland            728.562500   \n",
       "3             auto    Issaquah            720.500000   \n",
       "4             hold    Issaquah            693.470588   \n",
       "...            ...         ...                   ...   \n",
       "1087          hold     Colbert            728.868313   \n",
       "1088          hold   Kennewick            724.901985   \n",
       "1089          hold      Yakima            740.007218   \n",
       "1090          auto    Issaquah            762.409091   \n",
       "1091          hold    Issaquah            765.450000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      657.000000                   602.000000   \n",
       "1                      715.727273                   715.727273   \n",
       "2                      720.000000                   650.000000   \n",
       "3                      690.000000                   690.000000   \n",
       "4                      687.294118                   687.294118   \n",
       "...                           ...                          ...   \n",
       "1087                   735.566667                   650.000000   \n",
       "1088                   730.000000                   730.000000   \n",
       "1089                   763.828740                   687.930446   \n",
       "1090                   763.636364                   710.000000   \n",
       "1091                   760.000000                   710.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    50.0             False         False            False  \n",
       "1                     5.0             False         False             True  \n",
       "2                    70.0              True         False             True  \n",
       "3                    17.0             False         False             True  \n",
       "4                    17.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "1087                 20.0             False         False            False  \n",
       "1088                  0.0              True         False             True  \n",
       "1089                 25.0             False         False            False  \n",
       "1090                 10.0             False         False            False  \n",
       "1091                 10.0             False         False            False  \n",
       "\n",
       "[3612 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/aug/\" + file)\n",
    "    WA_aug = pd.concat([WA_aug, df])\n",
    "    \n",
    "WA_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_aug.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/WA-day/2017-dec-day-WA.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2968d640576fa151a8a4bae2c9a445efd1cd3de1</td>\n",
       "      <td>2017-12-24 07:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>WA</td>\n",
       "      <td>Buckley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bf856ad3b32ba9c8fb96f74d4dc0d3dadf5518ee</td>\n",
       "      <td>2017-12-21 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fircrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2234a1cc57e3f3272badb2d68217b38881bd0af7</td>\n",
       "      <td>2017-12-19 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fircrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a8f78becfdeafcd15e740373b6f71fe8a7e8b60d</td>\n",
       "      <td>2017-12-16 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>840</td>\n",
       "      <td>640</td>\n",
       "      <td>WA</td>\n",
       "      <td>Toppenish</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2234a1cc57e3f3272badb2d68217b38881bd0af7</td>\n",
       "      <td>2017-12-19 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>731</td>\n",
       "      <td>731</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fircrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391847</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2017-12-22 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>662</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391848</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-12-24 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391849</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2017-12-08 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391850</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-12-24 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>391851</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2017-12-17 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>760</td>\n",
       "      <td>702</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>375894 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2968d640576fa151a8a4bae2c9a445efd1cd3de1  2017-12-24 07:10:00 UTC   \n",
       "1       bf856ad3b32ba9c8fb96f74d4dc0d3dadf5518ee  2017-12-21 07:20:00 UTC   \n",
       "2       2234a1cc57e3f3272badb2d68217b38881bd0af7  2017-12-19 13:35:00 UTC   \n",
       "3       a8f78becfdeafcd15e740373b6f71fe8a7e8b60d  2017-12-16 19:20:00 UTC   \n",
       "4       2234a1cc57e3f3272badb2d68217b38881bd0af7  2017-12-19 19:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "391847  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2017-12-22 14:35:00 UTC   \n",
       "391848  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-12-24 15:35:00 UTC   \n",
       "391849  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2017-12-08 14:35:00 UTC   \n",
       "391850  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-12-24 16:45:00 UTC   \n",
       "391851  0c934c0fae5270491f44ada0ec919994dc79c6c0  2017-12-17 19:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               704                      755   \n",
       "1          heat          hold               714                      719   \n",
       "2          heat          hold               731                      731   \n",
       "3          auto          auto               687                      840   \n",
       "4          heat          hold               730                      731   \n",
       "...         ...           ...               ...                      ...   \n",
       "391847     heat          auto               662                      760   \n",
       "391848     auto          auto               694                      760   \n",
       "391849     heat          auto               681                      760   \n",
       "391850     auto          auto               712                      760   \n",
       "391851     auto          auto               715                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           705            WA         Buckley   \n",
       "1                           719            WA        Fircrest   \n",
       "2                           731            WA        Fircrest   \n",
       "3                           640            WA       Toppenish   \n",
       "4                           731            WA        Fircrest   \n",
       "...                         ...           ...             ...   \n",
       "391847                      660            WA  Spokane Valley   \n",
       "391848                      710            WA  Spokane Valley   \n",
       "391849                      680            WA  Spokane Valley   \n",
       "391850                      710            WA  Spokane Valley   \n",
       "391851                      702            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                        40             False         False            False   \n",
       "2                        40             False         False            False   \n",
       "3                        17              True         False             True   \n",
       "4                        40             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "391847                    7             False         False            False   \n",
       "391848                   40             False         False            False   \n",
       "391849                    7             False         False            False   \n",
       "391850                   40             False         False            False   \n",
       "391851                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "391847                       Gas  \n",
       "391848                       Gas  \n",
       "391849                       Gas  \n",
       "391850                       Gas  \n",
       "391851                       Gas  \n",
       "\n",
       "[375894 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/WA/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/WA-day/2018-dec-day-WA.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2234a1cc57e3f3272badb2d68217b38881bd0af7</td>\n",
       "      <td>2018-12-30 07:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>WA</td>\n",
       "      <td>Fircrest</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>59d38005a8042f63f116c018cde42bdcd5ad9334</td>\n",
       "      <td>2018-12-19 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>756</td>\n",
       "      <td>756</td>\n",
       "      <td>WA</td>\n",
       "      <td>Sumner</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>982d73694a28f9128af91f76cc248509be1e7a06</td>\n",
       "      <td>2018-12-18 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>707</td>\n",
       "      <td>670</td>\n",
       "      <td>WA</td>\n",
       "      <td>Moxee</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31fe11ab0b12531660a2e2e7882f5b691c93eff5</td>\n",
       "      <td>2018-12-31 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>655</td>\n",
       "      <td>801</td>\n",
       "      <td>720</td>\n",
       "      <td>WA</td>\n",
       "      <td>Yelm</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6f6c16cbfe77d7ebf9d680277566dee999e0d4d5</td>\n",
       "      <td>2018-12-12 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>766</td>\n",
       "      <td>716</td>\n",
       "      <td>WA</td>\n",
       "      <td>Selah</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574926</th>\n",
       "      <td>ab49fc85312897420c3f1f9eba9e0a303f3dc559</td>\n",
       "      <td>2018-12-22 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>652</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574927</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-12-04 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574928</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-12-08 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574929</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-12-27 17:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574930</th>\n",
       "      <td>0c934c0fae5270491f44ada0ec919994dc79c6c0</td>\n",
       "      <td>2018-12-12 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>537950 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       2234a1cc57e3f3272badb2d68217b38881bd0af7  2018-12-30 07:10:00 UTC   \n",
       "1       59d38005a8042f63f116c018cde42bdcd5ad9334  2018-12-19 07:15:00 UTC   \n",
       "2       982d73694a28f9128af91f76cc248509be1e7a06  2018-12-18 14:30:00 UTC   \n",
       "3       31fe11ab0b12531660a2e2e7882f5b691c93eff5  2018-12-31 18:50:00 UTC   \n",
       "4       6f6c16cbfe77d7ebf9d680277566dee999e0d4d5  2018-12-12 19:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "574926  ab49fc85312897420c3f1f9eba9e0a303f3dc559  2018-12-22 16:00:00 UTC   \n",
       "574927  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-12-04 17:55:00 UTC   \n",
       "574928  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-12-08 17:40:00 UTC   \n",
       "574929  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-12-27 17:25:00 UTC   \n",
       "574930  0c934c0fae5270491f44ada0ec919994dc79c6c0  2018-12-12 16:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               740                      742   \n",
       "1          heat          hold               754                      756   \n",
       "2          heat          auto               673                      707   \n",
       "3          heat          auto               655                      801   \n",
       "4          auto          hold               713                      766   \n",
       "...         ...           ...               ...                      ...   \n",
       "574926     auto          auto               652                      760   \n",
       "574927     auto          auto               705                      760   \n",
       "574928     auto          auto               699                      760   \n",
       "574929     auto          auto               700                      760   \n",
       "574930     auto          auto               707                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           742            WA        Fircrest   \n",
       "1                           756            WA          Sumner   \n",
       "2                           670            WA           Moxee   \n",
       "3                           720            WA            Yelm   \n",
       "4                           716            WA           Selah   \n",
       "...                         ...           ...             ...   \n",
       "574926                      650            WA  Spokane Valley   \n",
       "574927                      710            WA  Spokane Valley   \n",
       "574928                      710            WA  Spokane Valley   \n",
       "574929                      710            WA  Spokane Valley   \n",
       "574930                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "574926                    7             False         False            False   \n",
       "574927                   40             False         False            False   \n",
       "574928                   40             False         False            False   \n",
       "574929                   40             False         False            False   \n",
       "574930                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "574926                       Gas  \n",
       "574927                       Gas  \n",
       "574928                       Gas  \n",
       "574929                       Gas  \n",
       "574930                       Gas  \n",
       "\n",
       "[537950 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/WA/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/WA-day/2019-dec-day-WA.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0bd80b780d9ddea1ab114b066ceebba8e12dceb8</td>\n",
       "      <td>2019-12-06 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>717</td>\n",
       "      <td>717</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lake Tapps</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d087372e33a36ab54464198f8a69d5d7f799efae</td>\n",
       "      <td>2019-12-10 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>658</td>\n",
       "      <td>658</td>\n",
       "      <td>WA</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c184709ee30fbd739d5d1af3d9f4e95336e33f75</td>\n",
       "      <td>2019-12-29 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>763</td>\n",
       "      <td>713</td>\n",
       "      <td>WA</td>\n",
       "      <td>Rainier</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a84d9f4b2c7693647f53816e6a407dc8bcbcada9</td>\n",
       "      <td>2019-12-08 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>WA</td>\n",
       "      <td>Mill Creek</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5e142f843711dbcd89bb4f5f476ac1633e6815d1</td>\n",
       "      <td>2019-12-05 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>755</td>\n",
       "      <td>690</td>\n",
       "      <td>WA</td>\n",
       "      <td>Lynden</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623355</th>\n",
       "      <td>2db9bb8fd78474acbb163d049dc01c8887b2994e</td>\n",
       "      <td>2019-12-31 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623356</th>\n",
       "      <td>5f8e08c092689466ca938f8e22aecde60063d84b</td>\n",
       "      <td>2019-12-03 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>750</td>\n",
       "      <td>650</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623357</th>\n",
       "      <td>5f8e08c092689466ca938f8e22aecde60063d84b</td>\n",
       "      <td>2019-12-30 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>750</td>\n",
       "      <td>677</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623358</th>\n",
       "      <td>5f8e08c092689466ca938f8e22aecde60063d84b</td>\n",
       "      <td>2019-12-28 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>674</td>\n",
       "      <td>750</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>623359</th>\n",
       "      <td>138efe444f77e07c2ee37b86d8e12df20b7ab064</td>\n",
       "      <td>2019-12-02 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>593923 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       0bd80b780d9ddea1ab114b066ceebba8e12dceb8  2019-12-06 19:40:00 UTC   \n",
       "1       d087372e33a36ab54464198f8a69d5d7f799efae  2019-12-10 17:30:00 UTC   \n",
       "2       c184709ee30fbd739d5d1af3d9f4e95336e33f75  2019-12-29 16:15:00 UTC   \n",
       "4       a84d9f4b2c7693647f53816e6a407dc8bcbcada9  2019-12-08 18:40:00 UTC   \n",
       "5       5e142f843711dbcd89bb4f5f476ac1633e6815d1  2019-12-05 18:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "623355  2db9bb8fd78474acbb163d049dc01c8887b2994e  2019-12-31 17:40:00 UTC   \n",
       "623356  5f8e08c092689466ca938f8e22aecde60063d84b  2019-12-03 15:10:00 UTC   \n",
       "623357  5f8e08c092689466ca938f8e22aecde60063d84b  2019-12-30 07:25:00 UTC   \n",
       "623358  5f8e08c092689466ca938f8e22aecde60063d84b  2019-12-28 17:45:00 UTC   \n",
       "623359  138efe444f77e07c2ee37b86d8e12df20b7ab064  2019-12-02 17:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               714                      717   \n",
       "1          heat          hold               689                      658   \n",
       "2          auto          hold               705                      763   \n",
       "4          heat          hold               699                      698   \n",
       "5          auto          auto               716                      755   \n",
       "...         ...           ...               ...                      ...   \n",
       "623355     heat          hold               701                      740   \n",
       "623356     heat          auto               687                      750   \n",
       "623357     heat          auto               684                      750   \n",
       "623358     heat          auto               674                      750   \n",
       "623359     auto          hold               721                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           717            WA      Lake Tapps   \n",
       "1                           658            WA             NaN   \n",
       "2                           713            WA         Rainier   \n",
       "4                           698            WA      Mill Creek   \n",
       "5                           690            WA          Lynden   \n",
       "...                         ...           ...             ...   \n",
       "623355                      740            WA  Spokane Valley   \n",
       "623356                      650            WA  Spokane Valley   \n",
       "623357                      677            WA  Spokane Valley   \n",
       "623358                      660            WA  Spokane Valley   \n",
       "623359                      710            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                         0              True         False             True   \n",
       "2                        50             False         False             True   \n",
       "4                         0             False         False            False   \n",
       "5                        27             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "623355                   50              True         False            False   \n",
       "623356                   40              True         False            False   \n",
       "623357                   40              True         False            False   \n",
       "623358                   40              True         False            False   \n",
       "623359                    5             False         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "623355                       Gas  \n",
       "623356                       Gas  \n",
       "623357                       Gas  \n",
       "623358                       Gas  \n",
       "623359                  Electric  \n",
       "\n",
       "[593923 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/WA/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/WA-day/2020-dec-day-WA.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2020-12-26 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>637</td>\n",
       "      <td>693</td>\n",
       "      <td>693</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a</td>\n",
       "      <td>2020-12-05 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>WA</td>\n",
       "      <td>Colbert</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93949aace64e11360a850909aceeafd6d6f37900</td>\n",
       "      <td>2020-12-31 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>681</td>\n",
       "      <td>660</td>\n",
       "      <td>WA</td>\n",
       "      <td>Washougal</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9f93be13a3d66b0f019c730e48b8974cd8703bf9</td>\n",
       "      <td>2020-12-11 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>747</td>\n",
       "      <td>697</td>\n",
       "      <td>WA</td>\n",
       "      <td>Centralia</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>982d73694a28f9128af91f76cc248509be1e7a06</td>\n",
       "      <td>2020-12-03 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>727</td>\n",
       "      <td>680</td>\n",
       "      <td>WA</td>\n",
       "      <td>Moxee</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548343</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-12-01 12:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548344</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-12-01 08:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548345</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-12-01 07:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548346</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-12-01 11:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548347</th>\n",
       "      <td>c0522086d24320d88a6de0cfc363c999e6a7d61e</td>\n",
       "      <td>2020-12-01 09:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>WA</td>\n",
       "      <td>Spokane Valley</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>524701 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       93949aace64e11360a850909aceeafd6d6f37900  2020-12-26 17:25:00 UTC   \n",
       "1       fcb7d0ea0f57fb7544aadbc0e31620d53e44db8a  2020-12-05 17:45:00 UTC   \n",
       "3       93949aace64e11360a850909aceeafd6d6f37900  2020-12-31 14:15:00 UTC   \n",
       "4       9f93be13a3d66b0f019c730e48b8974cd8703bf9  2020-12-11 16:00:00 UTC   \n",
       "6       982d73694a28f9128af91f76cc248509be1e7a06  2020-12-03 14:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "548343  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-12-01 12:25:00 UTC   \n",
       "548344  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-12-01 08:05:00 UTC   \n",
       "548345  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-12-01 07:30:00 UTC   \n",
       "548346  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-12-01 11:10:00 UTC   \n",
       "548347  c0522086d24320d88a6de0cfc363c999e6a7d61e  2020-12-01 09:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               637                      693   \n",
       "1          heat          hold               701                      702   \n",
       "3          heat          hold               657                      681   \n",
       "4          auto          hold               696                      747   \n",
       "6          heat          auto               673                      727   \n",
       "...         ...           ...               ...                      ...   \n",
       "548343     heat          auto               745                      750   \n",
       "548344     heat          auto               751                      750   \n",
       "548345     heat          auto               746                      750   \n",
       "548346     heat          auto               745                      750   \n",
       "548347     heat          auto               755                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           693            WA       Washougal   \n",
       "1                           702            WA         Colbert   \n",
       "3                           660            WA       Washougal   \n",
       "4                           697            WA       Centralia   \n",
       "6                           680            WA           Moxee   \n",
       "...                         ...           ...             ...   \n",
       "548343                      750            WA  Spokane Valley   \n",
       "548344                      750            WA  Spokane Valley   \n",
       "548345                      750            WA  Spokane Valley   \n",
       "548346                      750            WA  Spokane Valley   \n",
       "548347                      750            WA  Spokane Valley   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        15             False         False            False   \n",
       "1                        20             False         False            False   \n",
       "3                        15             False         False            False   \n",
       "4                        90             False         False            False   \n",
       "6                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "548343                    5             False         False            False   \n",
       "548344                    5             False         False            False   \n",
       "548345                    5             False         False            False   \n",
       "548346                    5             False         False            False   \n",
       "548347                    5             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "548343                       Gas  \n",
       "548344                       Gas  \n",
       "548345                       Gas  \n",
       "548346                       Gas  \n",
       "548347                       Gas  \n",
       "\n",
       "[524701 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/WA/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/WA/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>019a1d7273650d66544a79b146b515ddfa3f0531</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>684.434783</td>\n",
       "      <td>721.043478</td>\n",
       "      <td>719.695652</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0232fcb7383720194f81e9ed746d72364c2b60b6</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>684.000000</td>\n",
       "      <td>771.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0232fcb7383720194f81e9ed746d72364c2b60b6</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>711.000000</td>\n",
       "      <td>682.000000</td>\n",
       "      <td>682.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0232fcb7383720194f81e9ed746d72364c2b60b6</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>698.750000</td>\n",
       "      <td>691.625000</td>\n",
       "      <td>691.625000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0232fcb7383720194f81e9ed746d72364c2b60b6</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Seattle</td>\n",
       "      <td>689.500000</td>\n",
       "      <td>701.366667</td>\n",
       "      <td>698.933333</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1338</th>\n",
       "      <td>fe7fec41cb28daf1bda70c75e42106ddf1526a4c</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>682.721053</td>\n",
       "      <td>771.197368</td>\n",
       "      <td>683.157895</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1339</th>\n",
       "      <td>fe8591238eb60be06f35ae8dded14551efdfad35</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>661.305556</td>\n",
       "      <td>700.791667</td>\n",
       "      <td>698.159722</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1340</th>\n",
       "      <td>feb039fb97c44493a5351770a1e1ae13d38959a5</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellingham</td>\n",
       "      <td>616.142857</td>\n",
       "      <td>732.142857</td>\n",
       "      <td>725.285714</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1341</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>695.232819</td>\n",
       "      <td>699.045348</td>\n",
       "      <td>699.045348</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1342</th>\n",
       "      <td>ffbba4969274f0403c05c7dca4fe68c2a9728fec</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tacoma</td>\n",
       "      <td>687.868687</td>\n",
       "      <td>757.474747</td>\n",
       "      <td>706.707071</td>\n",
       "      <td>50.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5443 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     019a1d7273650d66544a79b146b515ddfa3f0531   dec  2017     heat   \n",
       "1     0232fcb7383720194f81e9ed746d72364c2b60b6   dec  2017     auto   \n",
       "2     0232fcb7383720194f81e9ed746d72364c2b60b6   dec  2017     cool   \n",
       "3     0232fcb7383720194f81e9ed746d72364c2b60b6   dec  2017     cool   \n",
       "4     0232fcb7383720194f81e9ed746d72364c2b60b6   dec  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1338  fe7fec41cb28daf1bda70c75e42106ddf1526a4c   dec  2020     heat   \n",
       "1339  fe8591238eb60be06f35ae8dded14551efdfad35   dec  2020     heat   \n",
       "1340  feb039fb97c44493a5351770a1e1ae13d38959a5   dec  2020     heat   \n",
       "1341  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   dec  2020     heat   \n",
       "1342  ffbba4969274f0403c05c7dca4fe68c2a9728fec   dec  2020     auto   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold     Vancouver            684.434783   \n",
       "1             hold       Seattle            684.000000   \n",
       "2             auto       Seattle            711.000000   \n",
       "3             hold       Seattle            698.750000   \n",
       "4             auto       Seattle            689.500000   \n",
       "...            ...           ...                   ...   \n",
       "1338          auto      Issaquah            682.721053   \n",
       "1339          hold  Maple Valley            661.305556   \n",
       "1340          hold    Bellingham            616.142857   \n",
       "1341          hold        Yakima            695.232819   \n",
       "1342          hold        Tacoma            687.868687   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      721.043478                   719.695652   \n",
       "1                      771.000000                   690.000000   \n",
       "2                      682.000000                   682.000000   \n",
       "3                      691.625000                   691.625000   \n",
       "4                      701.366667                   698.933333   \n",
       "...                           ...                          ...   \n",
       "1338                   771.197368                   683.157895   \n",
       "1339                   700.791667                   698.159722   \n",
       "1340                   732.142857                   725.285714   \n",
       "1341                   699.045348                   699.045348   \n",
       "1342                   757.474747                   706.707071   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                    10.0             False         False            False  \n",
       "2                    10.0             False         False            False  \n",
       "3                    10.0             False         False            False  \n",
       "4                    10.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1338                 57.0             False         False            False  \n",
       "1339                 20.0             False         False            False  \n",
       "1340                 95.0             False         False            False  \n",
       "1341                 25.0             False         False            False  \n",
       "1342                 50.0              True         False             True  \n",
       "\n",
       "[5443 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/WA/dec/\" + file)\n",
    "    WA_dec = pd.concat([WA_dec, df])\n",
    "    \n",
    "WA_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_dec.to_csv(\"Scraper_Output/State_Month_Day/WA/WA_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/WA/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00fb48d1cbd52525aa51333669e22d5eae06f262</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Greenacres</td>\n",
       "      <td>693.000000</td>\n",
       "      <td>657.000000</td>\n",
       "      <td>602.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>019a1d7273650d66544a79b146b515ddfa3f0531</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>701.909091</td>\n",
       "      <td>715.727273</td>\n",
       "      <td>715.727273</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Richland</td>\n",
       "      <td>728.562500</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>70.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>720.500000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>038f6de02ac57361e4c383448667f66eff4e81f9</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>693.470588</td>\n",
       "      <td>687.294118</td>\n",
       "      <td>687.294118</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5276</th>\n",
       "      <td>fe8591238eb60be06f35ae8dded14551efdfad35</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maple Valley</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>677.400000</td>\n",
       "      <td>670.400000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5277</th>\n",
       "      <td>feb039fb97c44493a5351770a1e1ae13d38959a5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Bellingham</td>\n",
       "      <td>695.207792</td>\n",
       "      <td>650.363636</td>\n",
       "      <td>640.047619</td>\n",
       "      <td>95.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5278</th>\n",
       "      <td>fef8934c83e6b29016ef5b68074fc2d5a4da07f0</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Kennewick</td>\n",
       "      <td>733.960382</td>\n",
       "      <td>736.644901</td>\n",
       "      <td>736.597946</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5279</th>\n",
       "      <td>ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Yakima</td>\n",
       "      <td>713.561385</td>\n",
       "      <td>719.325103</td>\n",
       "      <td>719.325103</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5280</th>\n",
       "      <td>ff9d09ab72e971e34656b99dd7c9149bf12f7abe</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Issaquah</td>\n",
       "      <td>749.913043</td>\n",
       "      <td>769.891304</td>\n",
       "      <td>695.326087</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29216 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00fb48d1cbd52525aa51333669e22d5eae06f262   aug  2017     auto   \n",
       "1     019a1d7273650d66544a79b146b515ddfa3f0531   aug  2017     cool   \n",
       "2     02bcc0b8a0b37f0204b0d53afbe490b4f856b2bb   aug  2017     cool   \n",
       "3     038f6de02ac57361e4c383448667f66eff4e81f9   aug  2017     cool   \n",
       "4     038f6de02ac57361e4c383448667f66eff4e81f9   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "5276  fe8591238eb60be06f35ae8dded14551efdfad35   jun  2021     heat   \n",
       "5277  feb039fb97c44493a5351770a1e1ae13d38959a5   jun  2021     heat   \n",
       "5278  fef8934c83e6b29016ef5b68074fc2d5a4da07f0   jun  2021     cool   \n",
       "5279  ff1c1e3cdbfca9a5a91b1255072b7cd642f7e278   jun  2021     cool   \n",
       "5280  ff9d09ab72e971e34656b99dd7c9149bf12f7abe   jun  2021     auto   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold    Greenacres            693.000000   \n",
       "1             hold     Vancouver            701.909091   \n",
       "2             auto      Richland            728.562500   \n",
       "3             auto      Issaquah            720.500000   \n",
       "4             hold      Issaquah            693.470588   \n",
       "...            ...           ...                   ...   \n",
       "5276          hold  Maple Valley            670.000000   \n",
       "5277          hold    Bellingham            695.207792   \n",
       "5278          hold     Kennewick            733.960382   \n",
       "5279          hold        Yakima            713.561385   \n",
       "5280          hold      Issaquah            749.913043   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      657.000000                   602.000000   \n",
       "1                      715.727273                   715.727273   \n",
       "2                      720.000000                   650.000000   \n",
       "3                      690.000000                   690.000000   \n",
       "4                      687.294118                   687.294118   \n",
       "...                           ...                          ...   \n",
       "5276                   677.400000                   670.400000   \n",
       "5277                   650.363636                   640.047619   \n",
       "5278                   736.644901                   736.597946   \n",
       "5279                   719.325103                   719.325103   \n",
       "5280                   769.891304                   695.326087   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    50.0             False         False            False  \n",
       "1                     5.0             False         False             True  \n",
       "2                    70.0              True         False             True  \n",
       "3                    17.0             False         False             True  \n",
       "4                    17.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "5276                 20.0             False         False            False  \n",
       "5277                 95.0             False         False            False  \n",
       "5278                  0.0              True         False             True  \n",
       "5279                 25.0             False         False            False  \n",
       "5280                 10.0             False         False            False  \n",
       "\n",
       "[29216 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "WA_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/WA/\" + file)\n",
    "    WA_all = pd.concat([WA_all, df])\n",
    "    \n",
    "WA_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "WA_all.to_csv(\"Scraper_Output/State_Month_Day/WA_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['WA']\n",
      "Unique jan_2018: ['WA']\n",
      "Unique jan_2019: ['WA']\n",
      "Unique jan_2020: ['WA']\n",
      "Unique jan_2021: ['WA']\n",
      "Unique feb_2017: ['WA']\n",
      "Unique feb_2018: ['WA']\n",
      "Unique feb_2019: ['WA']\n",
      "Unique feb_2020: ['WA']\n",
      "Unique feb_2021: ['WA']\n",
      "Unique jun_2017: ['WA']\n",
      "Unique jun_2018: ['WA']\n",
      "Unique jun_2019: ['WA']\n",
      "Unique jun_2020: ['WA']\n",
      "Unique jun_2021: ['WA']\n",
      "Unique jul_2017: ['WA']\n",
      "Unique jul_2018: ['WA']\n",
      "Unique jul_2019: ['WA']\n",
      "Unique jul_2020: ['WA']\n",
      "Unique jul_2021: ['WA']\n",
      "Unique aug_2017: ['WA']\n",
      "Unique aug_2018: ['WA']\n",
      "Unique aug_2019: ['WA']\n",
      "Unique aug_2020: ['WA']\n",
      "Unique dec_2017: ['WA']\n",
      "Unique dec_2018: ['WA']\n",
      "Unique dec_2019: ['WA']\n",
      "Unique dec_2020: ['WA']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mWAe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}