DYD_OR-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/OR-day/2017-jan-day-OR.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-01-30 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-01-28 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-01-25 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-01-07 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-01-12 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115544</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-01-07 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115545</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-01-03 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115546</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-01-22 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115547</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-01-22 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115548</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-01-22 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>109441 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       dccaf065befac19f089838bd56e3d6ca78f374c6  2017-01-30 15:40:00 UTC   \n",
       "1       dccaf065befac19f089838bd56e3d6ca78f374c6  2017-01-28 19:35:00 UTC   \n",
       "2       dccaf065befac19f089838bd56e3d6ca78f374c6  2017-01-25 14:50:00 UTC   \n",
       "3       dccaf065befac19f089838bd56e3d6ca78f374c6  2017-01-07 18:30:00 UTC   \n",
       "4       dccaf065befac19f089838bd56e3d6ca78f374c6  2017-01-12 17:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "115544  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-01-07 16:30:00 UTC   \n",
       "115545  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-01-03 16:20:00 UTC   \n",
       "115546  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-01-22 18:30:00 UTC   \n",
       "115547  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-01-22 19:05:00 UTC   \n",
       "115548  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-01-22 18:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               695                      689   \n",
       "1          heat          hold               688                      689   \n",
       "2          heat          hold               685                      689   \n",
       "3          heat          hold               696                      698   \n",
       "4          heat          auto               704                      698   \n",
       "...         ...           ...               ...                      ...   \n",
       "115544     heat          hold               706                      700   \n",
       "115545     heat          hold               712                      710   \n",
       "115546     heat          hold               705                      710   \n",
       "115547     heat          hold               704                      710   \n",
       "115548     heat          hold               710                      710   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           689            OR     corvallis   \n",
       "1                           689            OR     corvallis   \n",
       "2                           689            OR     corvallis   \n",
       "3                           698            OR     corvallis   \n",
       "4                           698            OR     corvallis   \n",
       "...                         ...           ...           ...   \n",
       "115544                      700            OR  forest grove   \n",
       "115545                      710            OR  forest grove   \n",
       "115546                      710            OR  forest grove   \n",
       "115547                      710            OR  forest grove   \n",
       "115548                      710            OR  forest grove   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "115544                    0             False         False            False   \n",
       "115545                    0             False         False            False   \n",
       "115546                    0             False         False            False   \n",
       "115547                    0             False         False            False   \n",
       "115548                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "115544                       Gas  \n",
       "115545                       Gas  \n",
       "115546                       Gas  \n",
       "115547                       Gas  \n",
       "115548                       Gas  \n",
       "\n",
       "[109441 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>03caad454a66062fc56b9128d6de2ca5ae4b7996</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Portland</th>\n",
       "      <td>669.295455</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>05923e8afce99ca323843d899da2f0c48ba57cec</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Happy Valley</th>\n",
       "      <td>658.655303</td>\n",
       "      <td>655.276515</td>\n",
       "      <td>645.196970</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0624dc8fec23932f314f9e0f97a6c7fb059446b2</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Portland</th>\n",
       "      <td>719.402655</td>\n",
       "      <td>786.942478</td>\n",
       "      <td>725.176991</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Portland</th>\n",
       "      <td>724.582598</td>\n",
       "      <td>796.373310</td>\n",
       "      <td>732.052910</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0d0cf9141a32e41f86fe6fd7539c44bdfb535d74</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auxHeatOnly</th>\n",
       "      <th>hold</th>\n",
       "      <th>Coos Bay</th>\n",
       "      <td>669.458333</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>75.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Roseburg</th>\n",
       "      <td>669.812500</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Roseburg</th>\n",
       "      <td>684.888889</td>\n",
       "      <td>683.361111</td>\n",
       "      <td>679.777778</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Albany</th>\n",
       "      <td>684.099237</td>\n",
       "      <td>688.633588</td>\n",
       "      <td>687.961832</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">ff17c244fe42179fd802221bc9b6c177b3ad7d3e</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Beaverton</th>\n",
       "      <td>659.113689</td>\n",
       "      <td>819.415313</td>\n",
       "      <td>620.459397</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Beaverton</th>\n",
       "      <td>671.088235</td>\n",
       "      <td>669.264706</td>\n",
       "      <td>666.470588</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>324 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                            Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                 \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                669.295455   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley            658.655303   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                719.402655   \n",
       "                                                                hold          Portland                724.582598   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                669.458333   \n",
       "...                                                                                                          ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                669.812500   \n",
       "                                                                hold          Roseburg                684.888889   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                  684.099237   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton               659.113689   \n",
       "                                                                hold          Beaverton               671.088235   \n",
       "\n",
       "                                                                                            TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                        \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                       670.000000   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley                   655.276515   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                       786.942478   \n",
       "                                                                hold          Portland                       796.373310   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                       650.000000   \n",
       "...                                                                                                                 ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                       780.000000   \n",
       "                                                                hold          Roseburg                       683.361111   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                         688.633588   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton                      819.415313   \n",
       "                                                                hold          Beaverton                      669.264706   \n",
       "\n",
       "                                                                                            TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                        \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                       670.000000   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley                   645.196970   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                       725.176991   \n",
       "                                                                hold          Portland                       732.052910   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                       640.000000   \n",
       "...                                                                                                                 ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                       700.000000   \n",
       "                                                                hold          Roseburg                       679.777778   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                         687.961832   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton                      620.459397   \n",
       "                                                                hold          Beaverton                      666.470588   \n",
       "\n",
       "                                                                                            Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                      5.0   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley                 10.0   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                      0.0   \n",
       "                                                                hold          Portland                      0.0   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                     75.0   \n",
       "...                                                                                                         ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                     20.0   \n",
       "                                                                hold          Roseburg                     20.0   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                      120.0   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton                     5.0   \n",
       "                                                                hold          Beaverton                     5.0   \n",
       "\n",
       "                                                                                            allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                             \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                 False   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley             False   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                 False   \n",
       "                                                                hold          Portland                 False   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                  True   \n",
       "...                                                                                                      ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                 False   \n",
       "                                                                hold          Roseburg                 False   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                   False   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton                False   \n",
       "                                                                hold          Beaverton                False   \n",
       "\n",
       "                                                                                            Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                         \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland             False   \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley         False   \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland             False   \n",
       "                                                                hold          Portland             False   \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay             False   \n",
       "...                                                                                                  ...   \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg             False   \n",
       "                                                                hold          Roseburg             False   \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany               False   \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton            False   \n",
       "                                                                hold          Beaverton            False   \n",
       "\n",
       "                                                                                            Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                           \n",
       "03caad454a66062fc56b9128d6de2ca5ae4b7996 Jan   2017 heat        hold          Portland                False  \n",
       "05923e8afce99ca323843d899da2f0c48ba57cec Jan   2017 heat        hold          Happy Valley            False  \n",
       "0624dc8fec23932f314f9e0f97a6c7fb059446b2 Jan   2017 auto        auto          Portland                False  \n",
       "                                                                hold          Portland                False  \n",
       "0d0cf9141a32e41f86fe6fd7539c44bdfb535d74 Jan   2017 auxHeatOnly hold          Coos Bay                 True  \n",
       "...                                                                                                     ...  \n",
       "fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59 Jan   2017 heat        auto          Roseburg                 True  \n",
       "                                                                hold          Roseburg                 True  \n",
       "fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a Jan   2017 heat        hold          Albany                  False  \n",
       "ff17c244fe42179fd802221bc9b6c177b3ad7d3e Jan   2017 heat        auto          Beaverton               False  \n",
       "                                                                hold          Beaverton               False  \n",
       "\n",
       "[324 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/OR/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/OR-day/2018-jan-day-OR.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2018-01-16 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>694</td>\n",
       "      <td>719</td>\n",
       "      <td>699</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-01-20 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-01-20 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-01-20 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>653</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2018-01-15 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>719</td>\n",
       "      <td>699</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226620</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-01-28 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226621</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-01-23 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226622</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-01-18 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226623</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-01-23 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226624</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-01-28 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>211519 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       53a8afb9c2869da7d84a186b864ea3c571c2646c  2018-01-16 18:00:00 UTC   \n",
       "6       78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-01-20 17:10:00 UTC   \n",
       "11      78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-01-20 14:35:00 UTC   \n",
       "12      78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-01-20 14:55:00 UTC   \n",
       "16      53a8afb9c2869da7d84a186b864ea3c571c2646c  2018-01-15 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "226620  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-01-28 17:50:00 UTC   \n",
       "226621  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-01-23 19:50:00 UTC   \n",
       "226622  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-01-18 18:15:00 UTC   \n",
       "226623  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-01-23 17:50:00 UTC   \n",
       "226624  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-01-28 17:35:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               694                      719   \n",
       "6          heat          hold               657                      655   \n",
       "11         heat          hold               653                      655   \n",
       "12         heat          hold               653                      655   \n",
       "16         heat          auto               697                      719   \n",
       "...         ...           ...               ...                      ...   \n",
       "226620     heat          hold               707                      700   \n",
       "226621     heat          hold               696                      700   \n",
       "226622     heat          hold               701                      700   \n",
       "226623     heat          hold               689                      700   \n",
       "226624     heat          hold               702                      700   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           699            OR       Sheridan   \n",
       "6                           655            OR     The Dalles   \n",
       "11                          655            OR     The Dalles   \n",
       "12                          655            OR     The Dalles   \n",
       "16                          699            OR       Sheridan   \n",
       "...                         ...           ...            ...   \n",
       "226620                      700            OR  Klamath Falls   \n",
       "226621                      700            OR  Klamath Falls   \n",
       "226622                      700            OR  Klamath Falls   \n",
       "226623                      700            OR  Klamath Falls   \n",
       "226624                      700            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False             True   \n",
       "6                        55              True         False            False   \n",
       "11                       55              True         False            False   \n",
       "12                       55              True         False            False   \n",
       "16                       10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "226620                   29              True         False            False   \n",
       "226621                   29              True         False            False   \n",
       "226622                   29              True         False            False   \n",
       "226623                   29              True         False            False   \n",
       "226624                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "6                            Gas  \n",
       "11                           Gas  \n",
       "12                           Gas  \n",
       "16                      Electric  \n",
       "...                          ...  \n",
       "226620                       Gas  \n",
       "226621                       Gas  \n",
       "226622                       Gas  \n",
       "226623                       Gas  \n",
       "226624                       Gas  \n",
       "\n",
       "[211519 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/OR/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/OR-day/2019-jan-day-OR.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2019-01-11 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>761</td>\n",
       "      <td>761</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-01-09 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>752</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6</td>\n",
       "      <td>2019-01-03 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>648</td>\n",
       "      <td>840</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elkton</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2019-01-06 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>605</td>\n",
       "      <td>779</td>\n",
       "      <td>608</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>d667da3526d79b5b1165e06da5ea6013ccfb3f13</td>\n",
       "      <td>2019-01-08 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>775</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Saint Helens</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341786</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2019-01-20 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341787</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2019-01-20 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341788</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2019-01-26 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341789</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2019-01-26 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341790</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2019-01-20 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>314076 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c112fdef3cc591bfc328e7b5f040668e08165ca  2019-01-11 17:55:00 UTC   \n",
       "1       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-01-09 15:50:00 UTC   \n",
       "3       d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6  2019-01-03 17:00:00 UTC   \n",
       "7       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2019-01-06 17:25:00 UTC   \n",
       "8       d667da3526d79b5b1165e06da5ea6013ccfb3f13  2019-01-08 19:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "341786  69bce0cead73bc1724ae942dad09e3393da90e97  2019-01-20 18:20:00 UTC   \n",
       "341787  69bce0cead73bc1724ae942dad09e3393da90e97  2019-01-20 17:50:00 UTC   \n",
       "341788  69bce0cead73bc1724ae942dad09e3393da90e97  2019-01-26 18:00:00 UTC   \n",
       "341789  69bce0cead73bc1724ae942dad09e3393da90e97  2019-01-26 18:05:00 UTC   \n",
       "341790  69bce0cead73bc1724ae942dad09e3393da90e97  2019-01-20 16:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               756                      761   \n",
       "1          auto          hold               699                      752   \n",
       "3          auto          hold               648                      840   \n",
       "7          heat          auto               605                      779   \n",
       "8          heat          auto               700                      775   \n",
       "...         ...           ...               ...                      ...   \n",
       "341786     heat          hold               679                      725   \n",
       "341787     heat          hold               687                      725   \n",
       "341788     heat          hold               686                      725   \n",
       "341789     heat          hold               683                      725   \n",
       "341790     heat          hold               685                      725   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           761            OR   North Plains   \n",
       "1                           702            OR       Roseburg   \n",
       "3                           650            OR         Elkton   \n",
       "7                           608            OR       PORTLAND   \n",
       "8                           730            OR   Saint Helens   \n",
       "...                         ...           ...            ...   \n",
       "341786                      685            OR  Klamath Falls   \n",
       "341787                      685            OR  Klamath Falls   \n",
       "341788                      685            OR  Klamath Falls   \n",
       "341789                      685            OR  Klamath Falls   \n",
       "341790                      685            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                        90             False         False            False   \n",
       "3                        40              True         False             True   \n",
       "7                         7             False         False             True   \n",
       "8                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "341786                   40             False         False            False   \n",
       "341787                   40             False         False            False   \n",
       "341788                   40             False         False            False   \n",
       "341789                   40             False         False            False   \n",
       "341790                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                       Electric  \n",
       "7                       Electric  \n",
       "8                            Gas  \n",
       "...                          ...  \n",
       "341786                       Gas  \n",
       "341787                       Gas  \n",
       "341788                       Gas  \n",
       "341789                       Gas  \n",
       "341790                       Gas  \n",
       "\n",
       "[314076 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/OR/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/OR-day/2020-jan-day-OR.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9baeb67c105aa1a3213d6a3c4636f821257c2295</td>\n",
       "      <td>2020-01-10 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>662</td>\n",
       "      <td>785</td>\n",
       "      <td>675</td>\n",
       "      <td>OR</td>\n",
       "      <td>Irrigon</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>713f15c7203a2fc2488c73994d47d2077a399cec</td>\n",
       "      <td>2020-01-13 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>OR</td>\n",
       "      <td>Cornelius</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9baeb67c105aa1a3213d6a3c4636f821257c2295</td>\n",
       "      <td>2020-01-06 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>775</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Irrigon</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e214b1ed3bedd6d0a347f642ce888341b6bf025c</td>\n",
       "      <td>2020-01-11 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2020-01-08 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417444</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-01-12 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417445</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-01-07 07:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417446</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-01-07 07:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417447</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-01-07 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417448</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-01-31 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>692</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>394759 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       9baeb67c105aa1a3213d6a3c4636f821257c2295  2020-01-10 15:20:00 UTC   \n",
       "2       713f15c7203a2fc2488c73994d47d2077a399cec  2020-01-13 15:55:00 UTC   \n",
       "3       9baeb67c105aa1a3213d6a3c4636f821257c2295  2020-01-06 14:15:00 UTC   \n",
       "4       e214b1ed3bedd6d0a347f642ce888341b6bf025c  2020-01-11 15:25:00 UTC   \n",
       "5       53a8afb9c2869da7d84a186b864ea3c571c2646c  2020-01-08 15:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "417444  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-01-12 19:20:00 UTC   \n",
       "417445  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-01-07 07:00:00 UTC   \n",
       "417446  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-01-07 07:10:00 UTC   \n",
       "417447  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-01-07 07:05:00 UTC   \n",
       "417448  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-01-31 07:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          hold               662                      785   \n",
       "2          heat          hold               708                      716   \n",
       "3          auto          hold               700                      775   \n",
       "4          heat          hold               716                      722   \n",
       "5          heat          hold               679                      685   \n",
       "...         ...           ...               ...                      ...   \n",
       "417444     heat          hold               714                      720   \n",
       "417445     heat          hold               727                      730   \n",
       "417446     heat          hold               728                      730   \n",
       "417447     heat          hold               730                      730   \n",
       "417448     heat          hold               714                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           675            OR        Irrigon   \n",
       "2                           716            OR      Cornelius   \n",
       "3                           715            OR        Irrigon   \n",
       "4                           722            OR   North Plains   \n",
       "5                           685            OR       Sheridan   \n",
       "...                         ...           ...            ...   \n",
       "417444                      720            OR  Klamath Falls   \n",
       "417445                      730            OR  Klamath Falls   \n",
       "417446                      730            OR  Klamath Falls   \n",
       "417447                      730            OR  Klamath Falls   \n",
       "417448                      692            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        20             False         False            False   \n",
       "2                         0              True         False            False   \n",
       "3                        20             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "5                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "417444                   29              True         False            False   \n",
       "417445                   29              True         False            False   \n",
       "417446                   29              True         False            False   \n",
       "417447                   29              True         False            False   \n",
       "417448                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "417444                       Gas  \n",
       "417445                       Gas  \n",
       "417446                       Gas  \n",
       "417447                       Gas  \n",
       "417448                       Gas  \n",
       "\n",
       "[394759 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/OR/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/OR-day/2021-jan-day-OR.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93a6efa7b51384760480622702a81ca561056d45</td>\n",
       "      <td>2021-01-23 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>753</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central Point</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2021-01-11 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93a6efa7b51384760480622702a81ca561056d45</td>\n",
       "      <td>2021-01-23 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>753</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central Point</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32dfd347f85241965e760bd0d5bbc8255dabc80b</td>\n",
       "      <td>2021-01-25 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Shedd</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0d0cf9141a32e41f86fe6fd7539c44bdfb535d74</td>\n",
       "      <td>2021-01-08 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>OR</td>\n",
       "      <td>Coos Bay</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259157</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-01-13 07:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259158</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-01-01 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259159</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-01-19 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259160</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-01-12 07:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259161</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-01-12 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>247135 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       93a6efa7b51384760480622702a81ca561056d45  2021-01-23 18:50:00 UTC   \n",
       "1       114702ca6a8da3f42947dd25a6d56a0e60c41541  2021-01-11 19:50:00 UTC   \n",
       "2       93a6efa7b51384760480622702a81ca561056d45  2021-01-23 17:10:00 UTC   \n",
       "3       32dfd347f85241965e760bd0d5bbc8255dabc80b  2021-01-25 19:15:00 UTC   \n",
       "4       0d0cf9141a32e41f86fe6fd7539c44bdfb535d74  2021-01-08 17:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "259157  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-01-13 07:10:00 UTC   \n",
       "259158  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-01-01 07:15:00 UTC   \n",
       "259159  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-01-19 07:05:00 UTC   \n",
       "259160  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-01-12 07:00:00 UTC   \n",
       "259161  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-01-12 07:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               706                      753   \n",
       "1          heat          hold               668                      671   \n",
       "2          auto          hold               706                      753   \n",
       "3          heat          hold               687                      685   \n",
       "4          heat          hold               695                      691   \n",
       "...         ...           ...               ...                      ...   \n",
       "259157     heat          hold               729                      730   \n",
       "259158     heat          hold               723                      730   \n",
       "259159     heat          hold               723                      730   \n",
       "259160     heat          hold               722                      730   \n",
       "259161     heat          hold               729                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           710            OR  Central Point   \n",
       "1                           671            OR  Central point   \n",
       "2                           710            OR  Central Point   \n",
       "3                           685            OR          Shedd   \n",
       "4                           691            OR       Coos Bay   \n",
       "...                         ...           ...            ...   \n",
       "259157                      730            OR  Klamath Falls   \n",
       "259158                      730            OR  Klamath Falls   \n",
       "259159                      730            OR  Klamath Falls   \n",
       "259160                      730            OR  Klamath Falls   \n",
       "259161                      730            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         9             False         False            False   \n",
       "1                        55             False         False            False   \n",
       "2                         9             False         False            False   \n",
       "3                        27             False         False            False   \n",
       "4                        75              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "259157                   29              True         False            False   \n",
       "259158                   29              True         False            False   \n",
       "259159                   29              True         False            False   \n",
       "259160                   29              True         False            False   \n",
       "259161                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "259157                       Gas  \n",
       "259158                       Gas  \n",
       "259159                       Gas  \n",
       "259160                       Gas  \n",
       "259161                       Gas  \n",
       "\n",
       "[247135 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/OR/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>669.295455</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05923e8afce99ca323843d899da2f0c48ba57cec</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Happy Valley</td>\n",
       "      <td>658.655303</td>\n",
       "      <td>655.276515</td>\n",
       "      <td>645.196970</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>719.402655</td>\n",
       "      <td>786.942478</td>\n",
       "      <td>725.176991</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>724.582598</td>\n",
       "      <td>796.373310</td>\n",
       "      <td>732.052910</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0d0cf9141a32e41f86fe6fd7539c44bdfb535d74</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>hold</td>\n",
       "      <td>Coos Bay</td>\n",
       "      <td>669.458333</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>75.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>fd66b47e4d45c1c2ce2da22c440620fb85bc3d90</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Salem</td>\n",
       "      <td>672.972603</td>\n",
       "      <td>673.150685</td>\n",
       "      <td>673.150685</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>686.418960</td>\n",
       "      <td>684.351682</td>\n",
       "      <td>683.761468</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>White City</td>\n",
       "      <td>714.785714</td>\n",
       "      <td>721.714286</td>\n",
       "      <td>718.857143</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>697.237288</td>\n",
       "      <td>702.796610</td>\n",
       "      <td>702.508475</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>686.191667</td>\n",
       "      <td>687.258333</td>\n",
       "      <td>686.891667</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3359 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year     HvacMode  \\\n",
       "0    03caad454a66062fc56b9128d6de2ca5ae4b7996   Jan  2017         heat   \n",
       "1    05923e8afce99ca323843d899da2f0c48ba57cec   Jan  2017         heat   \n",
       "2    0624dc8fec23932f314f9e0f97a6c7fb059446b2   Jan  2017         auto   \n",
       "3    0624dc8fec23932f314f9e0f97a6c7fb059446b2   Jan  2017         auto   \n",
       "4    0d0cf9141a32e41f86fe6fd7539c44bdfb535d74   Jan  2017  auxHeatOnly   \n",
       "..                                        ...   ...   ...          ...   \n",
       "506  fd66b47e4d45c1c2ce2da22c440620fb85bc3d90   Jan  2021         heat   \n",
       "507  fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59   Jan  2021         heat   \n",
       "508  fe918247d6bdbd347613dd68f15c85893bb512da   Jan  2021         heat   \n",
       "509  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   Jan  2021         heat   \n",
       "510  ff38b7d2e7a775a99129867043a265bf5cbb48b2   Jan  2021         heat   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            hold      Portland            669.295455   \n",
       "1            hold  Happy Valley            658.655303   \n",
       "2            auto      Portland            719.402655   \n",
       "3            hold      Portland            724.582598   \n",
       "4            hold      Coos Bay            669.458333   \n",
       "..            ...           ...                   ...   \n",
       "506          hold         Salem            672.972603   \n",
       "507          hold      Roseburg            686.418960   \n",
       "508          hold    White City            714.785714   \n",
       "509          hold     Beaverton            697.237288   \n",
       "510          hold       Gresham            686.191667   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     670.000000                   670.000000   \n",
       "1                     655.276515                   645.196970   \n",
       "2                     786.942478                   725.176991   \n",
       "3                     796.373310                   732.052910   \n",
       "4                     650.000000                   640.000000   \n",
       "..                           ...                          ...   \n",
       "506                   673.150685                   673.150685   \n",
       "507                   684.351682                   683.761468   \n",
       "508                   721.714286                   718.857143   \n",
       "509                   702.796610                   702.508475   \n",
       "510                   687.258333                   686.891667   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False            False  \n",
       "1                   10.0             False         False            False  \n",
       "2                    0.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                   75.0              True         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "506                  0.0              True         False             True  \n",
       "507                 20.0             False         False             True  \n",
       "508                 57.0             False         False            False  \n",
       "509                  5.0             False         False            False  \n",
       "510                 20.0              True         False            False  \n",
       "\n",
       "[3359 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/jan/\" + file)\n",
    "    OR_jan = pd.concat([OR_jan, df])\n",
    "    \n",
    "OR_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_jan.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/OR-day/2017-feb-day-OR.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-02-05 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-02-16 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-02-21 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-02-23 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-02-07 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95492</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-02-17 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95493</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-02-03 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95494</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-02-04 15:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95495</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-02-04 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95496</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-02-07 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>89500 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-02-05 18:05:00 UTC   \n",
       "1      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-02-16 14:40:00 UTC   \n",
       "2      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-02-21 17:25:00 UTC   \n",
       "3      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-02-23 15:55:00 UTC   \n",
       "4      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-02-07 16:45:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "95492  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-02-17 17:10:00 UTC   \n",
       "95493  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-02-03 14:30:00 UTC   \n",
       "95494  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-02-04 15:25:00 UTC   \n",
       "95495  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-02-04 16:15:00 UTC   \n",
       "95496  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-02-07 17:05:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               693                      689   \n",
       "1         heat          auto               692                      689   \n",
       "2         heat          auto               689                      689   \n",
       "3         heat          auto               693                      689   \n",
       "4         heat          hold               689                      689   \n",
       "...        ...           ...               ...                      ...   \n",
       "95492     heat          hold               689                      700   \n",
       "95493     heat          hold               696                      700   \n",
       "95494     heat          hold               705                      700   \n",
       "95495     heat          hold               700                      700   \n",
       "95496     heat          hold               695                      700   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                          689            OR     corvallis   \n",
       "1                          689            OR     corvallis   \n",
       "2                          689            OR     corvallis   \n",
       "3                          689            OR     corvallis   \n",
       "4                          689            OR     corvallis   \n",
       "...                        ...           ...           ...   \n",
       "95492                      700            OR  forest grove   \n",
       "95493                      700            OR  forest grove   \n",
       "95494                      700            OR  forest grove   \n",
       "95495                      700            OR  forest grove   \n",
       "95496                      700            OR  forest grove   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        5             False         False            False   \n",
       "1                        5             False         False            False   \n",
       "2                        5             False         False            False   \n",
       "3                        5             False         False            False   \n",
       "4                        5             False         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "95492                    0             False         False            False   \n",
       "95493                    0             False         False            False   \n",
       "95494                    0             False         False            False   \n",
       "95495                    0             False         False            False   \n",
       "95496                    0             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "95492                       Gas  \n",
       "95493                       Gas  \n",
       "95494                       Gas  \n",
       "95495                       Gas  \n",
       "95496                       Gas  \n",
       "\n",
       "[89500 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/OR/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/OR-day/2018-feb-day-OR.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-02-09 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-02-09 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>78941a9bfdcd014723b1a9ddf3734e555ecea5ab</td>\n",
       "      <td>2018-02-11 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>OR</td>\n",
       "      <td>The Dalles</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c4640a78e89b6b196e0f0b8b91cbe978b502f0d7</td>\n",
       "      <td>2018-02-14 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>661</td>\n",
       "      <td>701</td>\n",
       "      <td>651</td>\n",
       "      <td>OR</td>\n",
       "      <td>Eagle Point</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>cd71171c504caf17e5e4786c93311b9ff08bdf66</td>\n",
       "      <td>2018-02-27 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>754</td>\n",
       "      <td>711</td>\n",
       "      <td>OR</td>\n",
       "      <td>Scappoose</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208375</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-02-18 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208376</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-02-17 07:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208377</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-02-18 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208378</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-02-18 07:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208379</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-02-04 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>192412 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-02-09 15:05:00 UTC   \n",
       "1       78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-02-09 17:45:00 UTC   \n",
       "6       78941a9bfdcd014723b1a9ddf3734e555ecea5ab  2018-02-11 16:05:00 UTC   \n",
       "8       c4640a78e89b6b196e0f0b8b91cbe978b502f0d7  2018-02-14 19:30:00 UTC   \n",
       "9       cd71171c504caf17e5e4786c93311b9ff08bdf66  2018-02-27 17:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "208375  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-02-18 07:20:00 UTC   \n",
       "208376  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-02-17 07:10:00 UTC   \n",
       "208377  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-02-18 07:15:00 UTC   \n",
       "208378  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-02-18 07:00:00 UTC   \n",
       "208379  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-02-04 07:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               660                      640   \n",
       "1          heat          hold               654                      640   \n",
       "6          heat          hold               640                      640   \n",
       "8          auto          auto               661                      701   \n",
       "9          heat          auto               715                      754   \n",
       "...         ...           ...               ...                      ...   \n",
       "208375     heat          hold               694                      700   \n",
       "208376     heat          hold               697                      700   \n",
       "208377     heat          hold               696                      700   \n",
       "208378     heat          hold               700                      700   \n",
       "208379     heat          hold               722                      720   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           640            OR     The Dalles   \n",
       "1                           640            OR     The Dalles   \n",
       "6                           640            OR     The Dalles   \n",
       "8                           651            OR    Eagle Point   \n",
       "9                           711            OR      Scappoose   \n",
       "...                         ...           ...            ...   \n",
       "208375                      700            OR  Klamath Falls   \n",
       "208376                      700            OR  Klamath Falls   \n",
       "208377                      700            OR  Klamath Falls   \n",
       "208378                      700            OR  Klamath Falls   \n",
       "208379                      720            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        55              True         False            False   \n",
       "1                        55              True         False            False   \n",
       "6                        55              True         False            False   \n",
       "8                        16             False         False            False   \n",
       "9                        27             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "208375                   29              True         False            False   \n",
       "208376                   29              True         False            False   \n",
       "208377                   29              True         False            False   \n",
       "208378                   29              True         False            False   \n",
       "208379                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "6                            Gas  \n",
       "8                            Gas  \n",
       "9                            Gas  \n",
       "...                          ...  \n",
       "208375                       Gas  \n",
       "208376                       Gas  \n",
       "208377                       Gas  \n",
       "208378                       Gas  \n",
       "208379                       Gas  \n",
       "\n",
       "[192412 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/OR/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/OR-day/2019-feb-day-OR.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1c927546884d866d0498c730035be2e4464967b2</td>\n",
       "      <td>2019-02-26 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>651</td>\n",
       "      <td>651</td>\n",
       "      <td>651</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lincoln City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-02-01 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>762</td>\n",
       "      <td>712</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d667da3526d79b5b1165e06da5ea6013ccfb3f13</td>\n",
       "      <td>2019-02-20 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>740</td>\n",
       "      <td>775</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>Saint Helens</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-02-24 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>752</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1c927546884d866d0498c730035be2e4464967b2</td>\n",
       "      <td>2019-02-22 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>647</td>\n",
       "      <td>651</td>\n",
       "      <td>651</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lincoln City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250395</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-02-16 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250396</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-02-17 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250397</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-02-17 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250398</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-02-10 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250399</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-02-10 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>700</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>233694 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       1c927546884d866d0498c730035be2e4464967b2  2019-02-26 19:25:00 UTC   \n",
       "2       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-02-01 18:35:00 UTC   \n",
       "3       d667da3526d79b5b1165e06da5ea6013ccfb3f13  2019-02-20 19:40:00 UTC   \n",
       "5       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-02-24 18:05:00 UTC   \n",
       "6       1c927546884d866d0498c730035be2e4464967b2  2019-02-22 15:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "250395  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-02-16 19:35:00 UTC   \n",
       "250396  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-02-17 07:15:00 UTC   \n",
       "250397  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-02-17 07:05:00 UTC   \n",
       "250398  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-02-10 16:50:00 UTC   \n",
       "250399  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-02-10 16:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          heat          hold               651                      651   \n",
       "2          auto          hold               707                      762   \n",
       "3          heat          auto               740                      775   \n",
       "5          auto          hold               699                      752   \n",
       "6          heat          hold               647                      651   \n",
       "...         ...           ...               ...                      ...   \n",
       "250395     heat          hold               709                      710   \n",
       "250396     heat          hold               716                      720   \n",
       "250397     heat          hold               718                      720   \n",
       "250398     heat          hold               704                      720   \n",
       "250399     heat          hold               700                      720   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           651            OR   Lincoln City   \n",
       "2                           712            OR       Roseburg   \n",
       "3                           750            OR   Saint Helens   \n",
       "5                           702            OR       Roseburg   \n",
       "6                           651            OR   Lincoln City   \n",
       "...                         ...           ...            ...   \n",
       "250395                      710            OR  Klamath Falls   \n",
       "250396                      720            OR  Klamath Falls   \n",
       "250397                      720            OR  Klamath Falls   \n",
       "250398                      720            OR  Klamath Falls   \n",
       "250399                      720            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         0             False         False            False   \n",
       "2                        90             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "5                        90             False         False            False   \n",
       "6                         0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "250395                   29              True         False            False   \n",
       "250396                   29              True         False            False   \n",
       "250397                   29              True         False            False   \n",
       "250398                   29              True         False            False   \n",
       "250399                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "250395                       Gas  \n",
       "250396                       Gas  \n",
       "250397                       Gas  \n",
       "250398                       Gas  \n",
       "250399                       Gas  \n",
       "\n",
       "[233694 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/OR/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/OR-day/2020-feb-day-OR.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2020-02-13 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>644</td>\n",
       "      <td>824</td>\n",
       "      <td>662</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9baeb67c105aa1a3213d6a3c4636f821257c2295</td>\n",
       "      <td>2020-02-28 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>OR</td>\n",
       "      <td>Irrigon</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2020-02-17 10:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2020-02-01 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>648</td>\n",
       "      <td>781</td>\n",
       "      <td>653</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2020-02-19 09:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361640</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-02-08 07:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361641</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-02-14 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>725</td>\n",
       "      <td>716</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361642</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-02-28 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361643</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2020-02-10 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>745</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361644</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2020-02-10 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>745</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>337840 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2020-02-13 15:50:00 UTC   \n",
       "1       9baeb67c105aa1a3213d6a3c4636f821257c2295  2020-02-28 17:40:00 UTC   \n",
       "2       53a8afb9c2869da7d84a186b864ea3c571c2646c  2020-02-17 10:05:00 UTC   \n",
       "3       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2020-02-01 16:30:00 UTC   \n",
       "5       53a8afb9c2869da7d84a186b864ea3c571c2646c  2020-02-19 09:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "361640  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-02-08 07:00:00 UTC   \n",
       "361641  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-02-14 07:25:00 UTC   \n",
       "361642  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-02-28 07:20:00 UTC   \n",
       "361643  69bce0cead73bc1724ae942dad09e3393da90e97  2020-02-10 15:40:00 UTC   \n",
       "361644  69bce0cead73bc1724ae942dad09e3393da90e97  2020-02-10 15:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               644                      824   \n",
       "1          auto          hold               721                      775   \n",
       "2          heat          hold               701                      699   \n",
       "3          heat          auto               648                      781   \n",
       "5          heat          hold               693                      699   \n",
       "...         ...           ...               ...                      ...   \n",
       "361640     heat          hold               720                      720   \n",
       "361641     heat          hold               714                      725   \n",
       "361642     heat          auto               723                      730   \n",
       "361643     heat          hold               681                      745   \n",
       "361644     heat          hold               679                      745   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           662            OR        Newberg   \n",
       "1                           725            OR        Irrigon   \n",
       "2                           699            OR       Sheridan   \n",
       "3                           653            OR       PORTLAND   \n",
       "5                           699            OR       Sheridan   \n",
       "...                         ...           ...            ...   \n",
       "361640                      720            OR  Klamath Falls   \n",
       "361641                      716            OR  Klamath Falls   \n",
       "361642                      730            OR  Klamath Falls   \n",
       "361643                      690            OR  Klamath Falls   \n",
       "361644                      690            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60              True         False             True   \n",
       "1                        20             False         False            False   \n",
       "2                        10             False         False             True   \n",
       "3                         7             False         False             True   \n",
       "5                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "361640                   29              True         False            False   \n",
       "361641                   29              True         False            False   \n",
       "361642                   29              True         False            False   \n",
       "361643                   40             False         False            False   \n",
       "361644                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "361640                       Gas  \n",
       "361641                       Gas  \n",
       "361642                       Gas  \n",
       "361643                       Gas  \n",
       "361644                       Gas  \n",
       "\n",
       "[337840 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/OR/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/OR-day/2021-feb-day-OR.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2021-02-02 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>df98dfa78799eecb3c6e4ec08abd443231822f1b</td>\n",
       "      <td>2021-02-11 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>617</td>\n",
       "      <td>683</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Silverton</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>df98dfa78799eecb3c6e4ec08abd443231822f1b</td>\n",
       "      <td>2021-02-12 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>616</td>\n",
       "      <td>683</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Silverton</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2021-02-04 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>779</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2021-02-05 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225312</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-02-10 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225313</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-02-18 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225314</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-02-04 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225315</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-02-05 07:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225316</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2021-02-04 07:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>219238 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       114702ca6a8da3f42947dd25a6d56a0e60c41541  2021-02-02 18:40:00 UTC   \n",
       "4       df98dfa78799eecb3c6e4ec08abd443231822f1b  2021-02-11 19:50:00 UTC   \n",
       "5       df98dfa78799eecb3c6e4ec08abd443231822f1b  2021-02-12 17:05:00 UTC   \n",
       "6       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2021-02-04 19:00:00 UTC   \n",
       "7       114702ca6a8da3f42947dd25a6d56a0e60c41541  2021-02-05 19:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "225312  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-02-10 07:20:00 UTC   \n",
       "225313  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-02-18 17:10:00 UTC   \n",
       "225314  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-02-04 07:15:00 UTC   \n",
       "225315  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-02-05 07:00:00 UTC   \n",
       "225316  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2021-02-04 07:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          heat          hold               668                      671   \n",
       "4          heat          hold               617                      683   \n",
       "5          heat          hold               616                      683   \n",
       "6          auto          hold               660                      779   \n",
       "7          heat          hold               655                      671   \n",
       "...         ...           ...               ...                      ...   \n",
       "225312     heat          hold               715                      720   \n",
       "225313     heat          hold               708                      720   \n",
       "225314     heat          hold               728                      730   \n",
       "225315     heat          hold               723                      730   \n",
       "225316     heat          hold               729                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           671            OR  Central point   \n",
       "4                           620            OR      Silverton   \n",
       "5                           620            OR      Silverton   \n",
       "6                           680            OR        Newberg   \n",
       "7                           671            OR  Central point   \n",
       "...                         ...           ...            ...   \n",
       "225312                      720            OR  Klamath Falls   \n",
       "225313                      720            OR  Klamath Falls   \n",
       "225314                      730            OR  Klamath Falls   \n",
       "225315                      730            OR  Klamath Falls   \n",
       "225316                      730            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        55             False         False            False   \n",
       "4                        90              True         False            False   \n",
       "5                        90              True         False            False   \n",
       "6                        60              True         False             True   \n",
       "7                        55             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "225312                   29              True         False            False   \n",
       "225313                   29              True         False            False   \n",
       "225314                   29              True         False            False   \n",
       "225315                   29              True         False            False   \n",
       "225316                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "6                       Electric  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "225312                       Gas  \n",
       "225313                       Gas  \n",
       "225314                       Gas  \n",
       "225315                       Gas  \n",
       "225316                       Gas  \n",
       "\n",
       "[219238 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/OR/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>680.234848</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>05923e8afce99ca323843d899da2f0c48ba57cec</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Happy Valley</td>\n",
       "      <td>667.117647</td>\n",
       "      <td>720.529412</td>\n",
       "      <td>700.941176</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>707.533235</td>\n",
       "      <td>782.225997</td>\n",
       "      <td>709.511078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>717.454790</td>\n",
       "      <td>791.557744</td>\n",
       "      <td>722.641898</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>099d727a71f6df3adc1add16bab7358831990e2c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>668.000000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>fd6dd3503a19a2386a09f4b25e636ef7cfb34eb3</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Albany</td>\n",
       "      <td>715.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>19.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>679.754266</td>\n",
       "      <td>680.795222</td>\n",
       "      <td>680.331058</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>White City</td>\n",
       "      <td>686.166667</td>\n",
       "      <td>675.000000</td>\n",
       "      <td>615.000000</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>493</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>695.854167</td>\n",
       "      <td>700.291667</td>\n",
       "      <td>699.854167</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>706.091837</td>\n",
       "      <td>703.668367</td>\n",
       "      <td>703.433673</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3251 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    03caad454a66062fc56b9128d6de2ca5ae4b7996   feb  2017     heat   \n",
       "1    05923e8afce99ca323843d899da2f0c48ba57cec   feb  2017     heat   \n",
       "2    0624dc8fec23932f314f9e0f97a6c7fb059446b2   feb  2017     auto   \n",
       "3    0624dc8fec23932f314f9e0f97a6c7fb059446b2   feb  2017     auto   \n",
       "4    099d727a71f6df3adc1add16bab7358831990e2c   feb  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "490  fd6dd3503a19a2386a09f4b25e636ef7cfb34eb3   feb  2021     heat   \n",
       "491  fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59   feb  2021     heat   \n",
       "492  fe918247d6bdbd347613dd68f15c85893bb512da   feb  2021     auto   \n",
       "493  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   feb  2021     heat   \n",
       "494  ff38b7d2e7a775a99129867043a265bf5cbb48b2   feb  2021     heat   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            hold      Portland            680.234848   \n",
       "1            hold  Happy Valley            667.117647   \n",
       "2            auto      Portland            707.533235   \n",
       "3            hold      Portland            717.454790   \n",
       "4            hold      Portland            668.000000   \n",
       "..            ...           ...                   ...   \n",
       "490          hold        Albany            715.000000   \n",
       "491          hold      Roseburg            679.754266   \n",
       "492          hold    White City            686.166667   \n",
       "493          hold     Beaverton            695.854167   \n",
       "494          hold       Gresham            706.091837   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     670.000000                   670.000000   \n",
       "1                     720.529412                   700.941176   \n",
       "2                     782.225997                   709.511078   \n",
       "3                     791.557744                   722.641898   \n",
       "4                     780.000000                   670.000000   \n",
       "..                           ...                          ...   \n",
       "490                   730.000000                   730.000000   \n",
       "491                   680.795222                   680.331058   \n",
       "492                   675.000000                   615.000000   \n",
       "493                   700.291667                   699.854167   \n",
       "494                   703.668367                   703.433673   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    5.0             False         False            False  \n",
       "1                   10.0             False         False            False  \n",
       "2                    0.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                  100.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "490                 19.0             False         False            False  \n",
       "491                 20.0             False         False             True  \n",
       "492                 57.0             False         False            False  \n",
       "493                  5.0             False         False            False  \n",
       "494                 20.0              True         False            False  \n",
       "\n",
       "[3251 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/feb/\" + file)\n",
    "    OR_feb = pd.concat([OR_feb, df])\n",
    "    \n",
    "OR_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_feb.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/OR-day/2017-jun-day-OR.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33</td>\n",
       "      <td>2017-06-03 15:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33</td>\n",
       "      <td>2017-06-03 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33</td>\n",
       "      <td>2017-06-05 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33</td>\n",
       "      <td>2017-06-02 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33</td>\n",
       "      <td>2017-06-03 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lebanon</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141742</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-06-21 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141743</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-06-22 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141744</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-06-22 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141745</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-06-22 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141746</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-06-21 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>130594 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33  2017-06-03 15:35:00 UTC   \n",
       "1       1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33  2017-06-03 13:30:00 UTC   \n",
       "2       1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33  2017-06-05 14:20:00 UTC   \n",
       "3       1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33  2017-06-02 17:05:00 UTC   \n",
       "4       1c828c9306ff8ba3fcd6c4c1df5839adb9b68a33  2017-06-03 15:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "141742  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-06-21 14:35:00 UTC   \n",
       "141743  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-06-22 16:15:00 UTC   \n",
       "141744  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-06-22 18:00:00 UTC   \n",
       "141745  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-06-22 19:30:00 UTC   \n",
       "141746  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-06-21 13:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               712                      765   \n",
       "1          auto          hold               728                      765   \n",
       "2          auto          hold               708                      765   \n",
       "3          auto          hold               715                      765   \n",
       "4          auto          hold               714                      765   \n",
       "...         ...           ...               ...                      ...   \n",
       "141742     cool          hold               683                      740   \n",
       "141743     cool          hold               693                      740   \n",
       "141744     cool          hold               694                      740   \n",
       "141745     cool          hold               705                      740   \n",
       "141746     cool          hold               687                      740   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           715            OR       Lebanon   \n",
       "1                           715            OR       Lebanon   \n",
       "2                           715            OR       Lebanon   \n",
       "3                           715            OR       Lebanon   \n",
       "4                           715            OR       Lebanon   \n",
       "...                         ...           ...           ...   \n",
       "141742                      740            OR  forest grove   \n",
       "141743                      740            OR  forest grove   \n",
       "141744                      740            OR  forest grove   \n",
       "141745                      740            OR  forest grove   \n",
       "141746                      740            OR  forest grove   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        47             False         False            False   \n",
       "1                        47             False         False            False   \n",
       "2                        47             False         False            False   \n",
       "3                        47             False         False            False   \n",
       "4                        47             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "141742                    0             False         False            False   \n",
       "141743                    0             False         False            False   \n",
       "141744                    0             False         False            False   \n",
       "141745                    0             False         False            False   \n",
       "141746                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "141742                       Gas  \n",
       "141743                       Gas  \n",
       "141744                       Gas  \n",
       "141745                       Gas  \n",
       "141746                       Gas  \n",
       "\n",
       "[130594 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/OR/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/OR-day/2018-jun-day-OR.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dd35868ee6a9a1ba55d65a6141fab45b29a4aac2</td>\n",
       "      <td>2018-06-04 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>699</td>\n",
       "      <td>669</td>\n",
       "      <td>OR</td>\n",
       "      <td>Cheshire</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dd35868ee6a9a1ba55d65a6141fab45b29a4aac2</td>\n",
       "      <td>2018-06-10 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>660</td>\n",
       "      <td>699</td>\n",
       "      <td>669</td>\n",
       "      <td>OR</td>\n",
       "      <td>Cheshire</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dd35868ee6a9a1ba55d65a6141fab45b29a4aac2</td>\n",
       "      <td>2018-06-08 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>699</td>\n",
       "      <td>669</td>\n",
       "      <td>OR</td>\n",
       "      <td>Cheshire</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dd35868ee6a9a1ba55d65a6141fab45b29a4aac2</td>\n",
       "      <td>2018-06-04 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>699</td>\n",
       "      <td>669</td>\n",
       "      <td>OR</td>\n",
       "      <td>Cheshire</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>65a70dce251bb809254faa8d86961902e4719cc0</td>\n",
       "      <td>2018-06-08 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>756</td>\n",
       "      <td>701</td>\n",
       "      <td>OR</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241389</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2018-06-30 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241390</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2018-06-30 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241391</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2018-06-30 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241392</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2018-06-30 17:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>738</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241393</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2018-06-30 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>738</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>213833 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       dd35868ee6a9a1ba55d65a6141fab45b29a4aac2  2018-06-04 18:20:00 UTC   \n",
       "1       dd35868ee6a9a1ba55d65a6141fab45b29a4aac2  2018-06-10 15:00:00 UTC   \n",
       "3       dd35868ee6a9a1ba55d65a6141fab45b29a4aac2  2018-06-08 18:35:00 UTC   \n",
       "4       dd35868ee6a9a1ba55d65a6141fab45b29a4aac2  2018-06-04 15:10:00 UTC   \n",
       "5       65a70dce251bb809254faa8d86961902e4719cc0  2018-06-08 19:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "241389  219fa6bfdeca2d936adab6a5d4394fb20284593f  2018-06-30 15:50:00 UTC   \n",
       "241390  219fa6bfdeca2d936adab6a5d4394fb20284593f  2018-06-30 16:25:00 UTC   \n",
       "241391  219fa6bfdeca2d936adab6a5d4394fb20284593f  2018-06-30 19:10:00 UTC   \n",
       "241392  219fa6bfdeca2d936adab6a5d4394fb20284593f  2018-06-30 17:55:00 UTC   \n",
       "241393  219fa6bfdeca2d936adab6a5d4394fb20284593f  2018-06-30 19:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               683                      699   \n",
       "1          auto          auto               660                      699   \n",
       "3          auto          auto               681                      699   \n",
       "4          auto          auto               673                      699   \n",
       "5          auto          hold               685                      756   \n",
       "...         ...           ...               ...                      ...   \n",
       "241389     auto          hold               731                      725   \n",
       "241390     auto          hold               727                      725   \n",
       "241391     auto          auto               734                      750   \n",
       "241392     auto          auto               738                      750   \n",
       "241393     auto          auto               738                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           669            OR       Cheshire   \n",
       "1                           669            OR       Cheshire   \n",
       "3                           669            OR       Cheshire   \n",
       "4                           669            OR       Cheshire   \n",
       "5                           701            OR        Ashland   \n",
       "...                         ...           ...            ...   \n",
       "241389                      675            OR  Klamath Falls   \n",
       "241390                      675            OR  Klamath Falls   \n",
       "241391                      700            OR  Klamath Falls   \n",
       "241392                      700            OR  Klamath Falls   \n",
       "241393                      700            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30              True         False             True   \n",
       "1                        30              True         False             True   \n",
       "3                        30              True         False             True   \n",
       "4                        30              True         False             True   \n",
       "5                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "241389                   17              True         False            False   \n",
       "241390                   17              True         False            False   \n",
       "241391                   17              True         False            False   \n",
       "241392                   17              True         False            False   \n",
       "241393                   17              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "241389                       Gas  \n",
       "241390                       Gas  \n",
       "241391                       Gas  \n",
       "241392                       Gas  \n",
       "241393                       Gas  \n",
       "\n",
       "[213833 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/OR/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/OR-day/2019-jun-day-OR.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e3af8fcbccf8a65ae71ec083381af39b3c7db7bd</td>\n",
       "      <td>2019-06-08 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>32676105dcefc3518965de07308b21cea587bcd8</td>\n",
       "      <td>2019-06-23 14:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>671</td>\n",
       "      <td>700</td>\n",
       "      <td>660</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ec990339e6a982961acf69687c69ebff9cf7cc57</td>\n",
       "      <td>2019-06-22 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1753a4cacf76f48dd0cc631192a93019f35d7e14</td>\n",
       "      <td>2019-06-16 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>700</td>\n",
       "      <td>630</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3ed300d9998e3b10a7aec9227feee362f5e17d1c</td>\n",
       "      <td>2019-06-01 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377790</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2019-06-27 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377791</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2019-06-27 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377792</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2019-06-08 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377793</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2019-06-08 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377794</th>\n",
       "      <td>219fa6bfdeca2d936adab6a5d4394fb20284593f</td>\n",
       "      <td>2019-06-08 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>17</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>346694 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e3af8fcbccf8a65ae71ec083381af39b3c7db7bd  2019-06-08 18:20:00 UTC   \n",
       "1       32676105dcefc3518965de07308b21cea587bcd8  2019-06-23 14:55:00 UTC   \n",
       "2       ec990339e6a982961acf69687c69ebff9cf7cc57  2019-06-22 16:10:00 UTC   \n",
       "3       1753a4cacf76f48dd0cc631192a93019f35d7e14  2019-06-16 14:15:00 UTC   \n",
       "4       3ed300d9998e3b10a7aec9227feee362f5e17d1c  2019-06-01 13:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "377790  219fa6bfdeca2d936adab6a5d4394fb20284593f  2019-06-27 19:00:00 UTC   \n",
       "377791  219fa6bfdeca2d936adab6a5d4394fb20284593f  2019-06-27 18:25:00 UTC   \n",
       "377792  219fa6bfdeca2d936adab6a5d4394fb20284593f  2019-06-08 16:55:00 UTC   \n",
       "377793  219fa6bfdeca2d936adab6a5d4394fb20284593f  2019-06-08 17:15:00 UTC   \n",
       "377794  219fa6bfdeca2d936adab6a5d4394fb20284593f  2019-06-08 15:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               693                      700   \n",
       "1          heat          auto               671                      700   \n",
       "2          heat          hold               697                      700   \n",
       "3          auto          hold               694                      700   \n",
       "4          heat          auto               698                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "377790     auto          auto               727                      760   \n",
       "377791     auto          auto               726                      760   \n",
       "377792     auto          auto               709                      760   \n",
       "377793     auto          auto               707                      760   \n",
       "377794     auto          auto               700                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           700            OR       Portland   \n",
       "1                           660            OR       Portland   \n",
       "2                           700            OR       Portland   \n",
       "3                           630            OR       Portland   \n",
       "4                           700            OR       Portland   \n",
       "...                         ...           ...            ...   \n",
       "377790                      700            OR  Klamath Falls   \n",
       "377791                      700            OR  Klamath Falls   \n",
       "377792                      710            OR  Klamath Falls   \n",
       "377793                      710            OR  Klamath Falls   \n",
       "377794                      710            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False             True   \n",
       "1                         0              True         False             True   \n",
       "2                        55              True         False             True   \n",
       "3                        20              True         False            False   \n",
       "4                        15             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "377790                   17              True         False            False   \n",
       "377791                   17              True         False            False   \n",
       "377792                   17              True         False            False   \n",
       "377793                   17              True         False            False   \n",
       "377794                   17              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "377790                       Gas  \n",
       "377791                       Gas  \n",
       "377792                       Gas  \n",
       "377793                       Gas  \n",
       "377794                       Gas  \n",
       "\n",
       "[346694 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/OR/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/OR-day/2020-jun-day-OR.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2020-06-19 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>743</td>\n",
       "      <td>662</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-06-16 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>df98dfa78799eecb3c6e4ec08abd443231822f1b</td>\n",
       "      <td>2020-06-15 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>753</td>\n",
       "      <td>733</td>\n",
       "      <td>OR</td>\n",
       "      <td>Silverton</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2020-06-02 11:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>717</td>\n",
       "      <td>667</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6ad7aa34fc964bff771cd12f54fa040aa436b610</td>\n",
       "      <td>2020-06-24 16:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>810</td>\n",
       "      <td>790</td>\n",
       "      <td>OR</td>\n",
       "      <td>Rogue River</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374921</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-06-20 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374922</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-06-20 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374923</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-06-20 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374924</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-06-02 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>374925</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-06-20 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>352878 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2020-06-19 16:50:00 UTC   \n",
       "1       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-06-16 19:00:00 UTC   \n",
       "2       df98dfa78799eecb3c6e4ec08abd443231822f1b  2020-06-15 16:30:00 UTC   \n",
       "3       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2020-06-02 11:15:00 UTC   \n",
       "4       6ad7aa34fc964bff771cd12f54fa040aa436b610  2020-06-24 16:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "374921  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-06-20 17:50:00 UTC   \n",
       "374922  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-06-20 17:15:00 UTC   \n",
       "374923  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-06-20 17:35:00 UTC   \n",
       "374924  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-06-02 16:45:00 UTC   \n",
       "374925  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-06-20 16:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               703                      743   \n",
       "1          heat          hold               657                      671   \n",
       "2          auto          hold               695                      753   \n",
       "3          auto          hold               681                      717   \n",
       "4          cool          hold               763                      810   \n",
       "...         ...           ...               ...                      ...   \n",
       "374921     cool          hold               742                      730   \n",
       "374922     cool          hold               750                      730   \n",
       "374923     cool          hold               746                      730   \n",
       "374924     cool          hold               725                      730   \n",
       "374925     cool          hold               739                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           662            OR        Newberg   \n",
       "1                           671            OR  Central point   \n",
       "2                           733            OR      Silverton   \n",
       "3                           667            OR       Roseburg   \n",
       "4                           790            OR    Rogue River   \n",
       "...                         ...           ...            ...   \n",
       "374921                      730            OR  Klamath Falls   \n",
       "374922                      730            OR  Klamath Falls   \n",
       "374923                      730            OR  Klamath Falls   \n",
       "374924                      730            OR  Klamath Falls   \n",
       "374925                      730            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60              True         False             True   \n",
       "1                        55             False         False            False   \n",
       "2                        90              True         False            False   \n",
       "3                        90             False         False            False   \n",
       "4                        50             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "374921                   29              True         False            False   \n",
       "374922                   29              True         False            False   \n",
       "374923                   29              True         False            False   \n",
       "374924                   29              True         False            False   \n",
       "374925                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "374921                       Gas  \n",
       "374922                       Gas  \n",
       "374923                       Gas  \n",
       "374924                       Gas  \n",
       "374925                       Gas  \n",
       "\n",
       "[352878 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/OR/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/OR-day/2021-jun-day-OR.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8f0cff68721f61623e70c5ab7f16efaea52c1581</td>\n",
       "      <td>2021-06-11 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elmira</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8f0cff68721f61623e70c5ab7f16efaea52c1581</td>\n",
       "      <td>2021-06-06 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elmira</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8f0cff68721f61623e70c5ab7f16efaea52c1581</td>\n",
       "      <td>2021-06-06 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elmira</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0d0cf9141a32e41f86fe6fd7539c44bdfb535d74</td>\n",
       "      <td>2021-06-24 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Coos Bay</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0d0cf9141a32e41f86fe6fd7539c44bdfb535d74</td>\n",
       "      <td>2021-06-05 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Coos Bay</td>\n",
       "      <td>75</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235560</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2021-06-05 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235561</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2021-06-05 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235562</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2021-06-06 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>644</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235563</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2021-06-05 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>235564</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2021-06-06 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>644</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>225747 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       8f0cff68721f61623e70c5ab7f16efaea52c1581  2021-06-11 19:20:00 UTC   \n",
       "2       8f0cff68721f61623e70c5ab7f16efaea52c1581  2021-06-06 19:50:00 UTC   \n",
       "3       8f0cff68721f61623e70c5ab7f16efaea52c1581  2021-06-06 19:15:00 UTC   \n",
       "4       0d0cf9141a32e41f86fe6fd7539c44bdfb535d74  2021-06-24 16:00:00 UTC   \n",
       "5       0d0cf9141a32e41f86fe6fd7539c44bdfb535d74  2021-06-05 17:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "235560  e8cd29c205dfbd5aa461efb3de976b243a38835f  2021-06-05 16:40:00 UTC   \n",
       "235561  e8cd29c205dfbd5aa461efb3de976b243a38835f  2021-06-05 15:45:00 UTC   \n",
       "235562  e8cd29c205dfbd5aa461efb3de976b243a38835f  2021-06-06 17:40:00 UTC   \n",
       "235563  e8cd29c205dfbd5aa461efb3de976b243a38835f  2021-06-05 14:45:00 UTC   \n",
       "235564  e8cd29c205dfbd5aa461efb3de976b243a38835f  2021-06-06 15:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          heat          hold               706                      680   \n",
       "2          heat          hold               690                      680   \n",
       "3          heat          hold               686                      680   \n",
       "4          heat          hold               686                      690   \n",
       "5          heat          hold               694                      690   \n",
       "...         ...           ...               ...                      ...   \n",
       "235560     auto          hold               692                      760   \n",
       "235561     auto          hold               691                      760   \n",
       "235562     auto          hold               644                      760   \n",
       "235563     auto          hold               693                      760   \n",
       "235564     auto          hold               644                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "1                           680            OR     Elmira                   75   \n",
       "2                           680            OR     Elmira                   75   \n",
       "3                           680            OR     Elmira                   75   \n",
       "4                           690            OR   Coos Bay                   75   \n",
       "5                           690            OR   Coos Bay                   75   \n",
       "...                         ...           ...        ...                  ...   \n",
       "235560                      650            OR  Gladstone                  100   \n",
       "235561                      650            OR  Gladstone                  100   \n",
       "235562                      650            OR  Gladstone                  100   \n",
       "235563                      650            OR  Gladstone                  100   \n",
       "235564                      650            OR  Gladstone                  100   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                   True         False             True   \n",
       "5                   True         False             True   \n",
       "...                  ...           ...              ...   \n",
       "235560             False         False            False   \n",
       "235561             False         False            False   \n",
       "235562             False         False            False   \n",
       "235563             False         False            False   \n",
       "235564             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "5                       Electric  \n",
       "...                          ...  \n",
       "235560                       Gas  \n",
       "235561                       Gas  \n",
       "235562                       Gas  \n",
       "235563                       Gas  \n",
       "235564                       Gas  \n",
       "\n",
       "[225747 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/OR/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006b3b1d03de005725492594b7733c3ef67d7dfe</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oregon City</td>\n",
       "      <td>714.644068</td>\n",
       "      <td>748.254237</td>\n",
       "      <td>694.186441</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00b089459452edc92a0469d6ea9d74767924f3f5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>669.036893</td>\n",
       "      <td>761.382524</td>\n",
       "      <td>761.079612</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>727.666667</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>645.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>747.333333</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>052a8daf4686b3a7f4a807ef3239f57490eb40d2</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Forest Grove</td>\n",
       "      <td>715.320513</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>726</th>\n",
       "      <td>fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>761.627907</td>\n",
       "      <td>756.250000</td>\n",
       "      <td>756.366279</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>727</th>\n",
       "      <td>fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Albany</td>\n",
       "      <td>670.222222</td>\n",
       "      <td>773.777778</td>\n",
       "      <td>773.777778</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>White City</td>\n",
       "      <td>691.432836</td>\n",
       "      <td>679.552239</td>\n",
       "      <td>679.059701</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>712.991935</td>\n",
       "      <td>708.838710</td>\n",
       "      <td>708.766129</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>730</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>712.201278</td>\n",
       "      <td>723.092652</td>\n",
       "      <td>723.035144</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3881 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    006b3b1d03de005725492594b7733c3ef67d7dfe   jun  2017     auto   \n",
       "1    00b089459452edc92a0469d6ea9d74767924f3f5   jun  2017     cool   \n",
       "2    03caad454a66062fc56b9128d6de2ca5ae4b7996   jun  2017     auto   \n",
       "3    03caad454a66062fc56b9128d6de2ca5ae4b7996   jun  2017     auto   \n",
       "4    052a8daf4686b3a7f4a807ef3239f57490eb40d2   jun  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "726  fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59   jun  2021     cool   \n",
       "727  fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a   jun  2021     cool   \n",
       "728  fe918247d6bdbd347613dd68f15c85893bb512da   jun  2021     cool   \n",
       "729  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   jun  2021     cool   \n",
       "730  ff38b7d2e7a775a99129867043a265bf5cbb48b2   jun  2021     cool   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            auto   Oregon City            714.644068   \n",
       "1            hold     Beaverton            669.036893   \n",
       "2            auto      Portland            727.666667   \n",
       "3            hold      Portland            747.333333   \n",
       "4            hold  Forest Grove            715.320513   \n",
       "..            ...           ...                   ...   \n",
       "726          hold      Roseburg            761.627907   \n",
       "727          hold        Albany            670.222222   \n",
       "728          hold    White City            691.432836   \n",
       "729          hold     Beaverton            712.991935   \n",
       "730          hold       Gresham            712.201278   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     748.254237                   694.186441   \n",
       "1                     761.382524                   761.079612   \n",
       "2                     750.000000                   645.000000   \n",
       "3                     760.000000                   670.000000   \n",
       "4                     730.000000                   730.000000   \n",
       "..                           ...                          ...   \n",
       "726                   756.250000                   756.366279   \n",
       "727                   773.777778                   773.777778   \n",
       "728                   679.552239                   679.059701   \n",
       "729                   708.838710                   708.766129   \n",
       "730                   723.092652                   723.035144   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   25.0              True         False            False  \n",
       "2                    5.0             False         False            False  \n",
       "3                    5.0             False         False            False  \n",
       "4                   10.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "726                 20.0             False         False             True  \n",
       "727                120.0             False         False            False  \n",
       "728                 57.0             False         False            False  \n",
       "729                  5.0             False         False            False  \n",
       "730                 20.0              True         False            False  \n",
       "\n",
       "[3881 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/jun/\" + file)\n",
    "    OR_jun = pd.concat([OR_jun, df])\n",
    "    \n",
    "OR_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_jun.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/OR-day/2017-jul-day-OR.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>368592eef613ea59348ff70f972c088ff48bd92f</td>\n",
       "      <td>2017-07-06 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>709</td>\n",
       "      <td>659</td>\n",
       "      <td>OR</td>\n",
       "      <td>Aloha</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9248bc8a2f1e92e98edb4633dca150eca8f07d6f</td>\n",
       "      <td>2017-07-22 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>810</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Corvallis</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9248bc8a2f1e92e98edb4633dca150eca8f07d6f</td>\n",
       "      <td>2017-07-22 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>810</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Corvallis</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6ad7aa34fc964bff771cd12f54fa040aa436b610</td>\n",
       "      <td>2017-07-05 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>810</td>\n",
       "      <td>790</td>\n",
       "      <td>OR</td>\n",
       "      <td>Rogue River</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9248bc8a2f1e92e98edb4633dca150eca8f07d6f</td>\n",
       "      <td>2017-07-22 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>810</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Corvallis</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164655</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-07-21 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164656</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-07-20 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164657</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-07-01 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164658</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-07-13 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164659</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-07-13 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>152369 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       368592eef613ea59348ff70f972c088ff48bd92f  2017-07-06 18:05:00 UTC   \n",
       "1       9248bc8a2f1e92e98edb4633dca150eca8f07d6f  2017-07-22 17:40:00 UTC   \n",
       "2       9248bc8a2f1e92e98edb4633dca150eca8f07d6f  2017-07-22 16:50:00 UTC   \n",
       "3       6ad7aa34fc964bff771cd12f54fa040aa436b610  2017-07-05 17:05:00 UTC   \n",
       "4       9248bc8a2f1e92e98edb4633dca150eca8f07d6f  2017-07-22 17:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "164655  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-07-21 18:45:00 UTC   \n",
       "164656  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-07-20 19:50:00 UTC   \n",
       "164657  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-07-01 17:40:00 UTC   \n",
       "164658  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-07-13 17:50:00 UTC   \n",
       "164659  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-07-13 18:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               706                      709   \n",
       "1          cool          hold               731                      810   \n",
       "2          cool          hold               727                      810   \n",
       "3          cool          hold               766                      810   \n",
       "4          cool          hold               727                      810   \n",
       "...         ...           ...               ...                      ...   \n",
       "164655     cool          hold               704                      750   \n",
       "164656     cool          hold               708                      750   \n",
       "164657     cool          hold               701                      750   \n",
       "164658     cool          hold               722                      760   \n",
       "164659     cool          hold               714                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           659            OR         Aloha   \n",
       "1                           740            OR     Corvallis   \n",
       "2                           740            OR     Corvallis   \n",
       "3                           790            OR   Rogue River   \n",
       "4                           740            OR     Corvallis   \n",
       "...                         ...           ...           ...   \n",
       "164655                      750            OR  forest grove   \n",
       "164656                      750            OR  forest grove   \n",
       "164657                      750            OR  forest grove   \n",
       "164658                      760            OR  forest grove   \n",
       "164659                      760            OR  forest grove   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        40             False         False            False   \n",
       "1                        40              True         False            False   \n",
       "2                        40              True         False            False   \n",
       "3                        50             False         False             True   \n",
       "4                        40              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "164655                    0             False         False            False   \n",
       "164656                    0             False         False            False   \n",
       "164657                    0             False         False            False   \n",
       "164658                    0             False         False            False   \n",
       "164659                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "164655                       Gas  \n",
       "164656                       Gas  \n",
       "164657                       Gas  \n",
       "164658                       Gas  \n",
       "164659                       Gas  \n",
       "\n",
       "[152369 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/OR/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/OR-day/2018-jul-day-OR.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65a70dce251bb809254faa8d86961902e4719cc0</td>\n",
       "      <td>2018-07-17 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>665</td>\n",
       "      <td>605</td>\n",
       "      <td>OR</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>65a70dce251bb809254faa8d86961902e4719cc0</td>\n",
       "      <td>2018-07-16 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>665</td>\n",
       "      <td>615</td>\n",
       "      <td>OR</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>65a70dce251bb809254faa8d86961902e4719cc0</td>\n",
       "      <td>2018-07-17 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>662</td>\n",
       "      <td>665</td>\n",
       "      <td>605</td>\n",
       "      <td>OR</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d32546856f82383cac4ce8007c048507ca518c28</td>\n",
       "      <td>2018-07-14 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>795</td>\n",
       "      <td>615</td>\n",
       "      <td>OR</td>\n",
       "      <td>oak grove</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>65a70dce251bb809254faa8d86961902e4719cc0</td>\n",
       "      <td>2018-07-17 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>665</td>\n",
       "      <td>605</td>\n",
       "      <td>OR</td>\n",
       "      <td>Ashland</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322821</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-07-28 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322822</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-07-28 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322823</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-07-28 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322824</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-07-28 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322825</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-07-28 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>279075 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       65a70dce251bb809254faa8d86961902e4719cc0  2018-07-17 16:10:00 UTC   \n",
       "1       65a70dce251bb809254faa8d86961902e4719cc0  2018-07-16 17:05:00 UTC   \n",
       "2       65a70dce251bb809254faa8d86961902e4719cc0  2018-07-17 18:50:00 UTC   \n",
       "3       d32546856f82383cac4ce8007c048507ca518c28  2018-07-14 18:40:00 UTC   \n",
       "4       65a70dce251bb809254faa8d86961902e4719cc0  2018-07-17 17:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "322821  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-07-28 18:50:00 UTC   \n",
       "322822  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-07-28 18:25:00 UTC   \n",
       "322823  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-07-28 17:45:00 UTC   \n",
       "322824  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-07-28 17:35:00 UTC   \n",
       "322825  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-07-28 17:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               659                      665   \n",
       "1          auto          hold               666                      665   \n",
       "2          auto          hold               662                      665   \n",
       "3          auto          hold               731                      795   \n",
       "4          auto          hold               669                      665   \n",
       "...         ...           ...               ...                      ...   \n",
       "322821     auto          hold               743                      760   \n",
       "322822     auto          hold               741                      760   \n",
       "322823     auto          hold               730                      760   \n",
       "322824     auto          hold               728                      760   \n",
       "322825     auto          hold               730                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           605            OR        Ashland   \n",
       "1                           615            OR        Ashland   \n",
       "2                           605            OR        Ashland   \n",
       "3                           615            OR      oak grove   \n",
       "4                           605            OR        Ashland   \n",
       "...                         ...           ...            ...   \n",
       "322821                      710            OR  Klamath Falls   \n",
       "322822                      710            OR  Klamath Falls   \n",
       "322823                      710            OR  Klamath Falls   \n",
       "322824                      710            OR  Klamath Falls   \n",
       "322825                      710            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                         0              True         False            False   \n",
       "2                         0              True         False            False   \n",
       "3                        50             False         False            False   \n",
       "4                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "322821                   29              True         False            False   \n",
       "322822                   29              True         False            False   \n",
       "322823                   29              True         False            False   \n",
       "322824                   29              True         False            False   \n",
       "322825                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "322821                       Gas  \n",
       "322822                       Gas  \n",
       "322823                       Gas  \n",
       "322824                       Gas  \n",
       "322825                       Gas  \n",
       "\n",
       "[279075 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/OR/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/OR-day/2019-jul-day-OR.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>df98dfa78799eecb3c6e4ec08abd443231822f1b</td>\n",
       "      <td>2019-07-31 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>OR</td>\n",
       "      <td>Silverton</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8ad786ec06c05d4ae243023be9b45144a11db7c4</td>\n",
       "      <td>2019-07-24 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>635</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>OR</td>\n",
       "      <td>Canyon City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>876b9dfe5bf0540d52e634786e681978524b51a1</td>\n",
       "      <td>2019-07-21 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>784</td>\n",
       "      <td>747</td>\n",
       "      <td>OR</td>\n",
       "      <td>McMinnville</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-07-17 14:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>738</td>\n",
       "      <td>688</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>876b9dfe5bf0540d52e634786e681978524b51a1</td>\n",
       "      <td>2019-07-16 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>784</td>\n",
       "      <td>747</td>\n",
       "      <td>OR</td>\n",
       "      <td>McMinnville</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409558</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-07-13 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409559</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-07-15 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409560</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-07-12 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409561</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-07-13 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409562</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-07-15 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>373489 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       df98dfa78799eecb3c6e4ec08abd443231822f1b  2019-07-31 19:10:00 UTC   \n",
       "1       8ad786ec06c05d4ae243023be9b45144a11db7c4  2019-07-24 14:20:00 UTC   \n",
       "2       876b9dfe5bf0540d52e634786e681978524b51a1  2019-07-21 19:15:00 UTC   \n",
       "3       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-07-17 14:35:00 UTC   \n",
       "4       876b9dfe5bf0540d52e634786e681978524b51a1  2019-07-16 18:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "409558  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-07-13 18:30:00 UTC   \n",
       "409559  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-07-15 18:00:00 UTC   \n",
       "409560  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-07-12 18:15:00 UTC   \n",
       "409561  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-07-13 17:35:00 UTC   \n",
       "409562  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-07-15 19:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      743   \n",
       "1          cool          auto               635                      630   \n",
       "2          cool          hold               696                      784   \n",
       "3          auto          hold               710                      738   \n",
       "4          cool          hold               693                      784   \n",
       "...         ...           ...               ...                      ...   \n",
       "409558     auto          hold               753                      750   \n",
       "409559     auto          hold               747                      750   \n",
       "409560     auto          auto               752                      750   \n",
       "409561     auto          hold               761                      750   \n",
       "409562     auto          hold               742                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           743            OR      Silverton   \n",
       "1                           630            OR    Canyon City   \n",
       "2                           747            OR    McMinnville   \n",
       "3                           688            OR       Roseburg   \n",
       "4                           747            OR    McMinnville   \n",
       "...                         ...           ...            ...   \n",
       "409558                      680            OR  Klamath Falls   \n",
       "409559                      680            OR  Klamath Falls   \n",
       "409560                      680            OR  Klamath Falls   \n",
       "409561                      680            OR  Klamath Falls   \n",
       "409562                      680            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        90              True         False            False   \n",
       "1                        10             False         False            False   \n",
       "2                        10              True         False             True   \n",
       "3                        90             False         False            False   \n",
       "4                        10              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "409558                   29              True         False            False   \n",
       "409559                   29              True         False            False   \n",
       "409560                   29              True         False            False   \n",
       "409561                   29              True         False            False   \n",
       "409562                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "409558                       Gas  \n",
       "409559                       Gas  \n",
       "409560                       Gas  \n",
       "409561                       Gas  \n",
       "409562                       Gas  \n",
       "\n",
       "[373489 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/OR/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/OR-day/2020-jul-day-OR.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c4640a78e89b6b196e0f0b8b91cbe978b502f0d7</td>\n",
       "      <td>2020-07-15 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>628</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Eagle Point</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2020-07-20 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>761</td>\n",
       "      <td>635</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c4640a78e89b6b196e0f0b8b91cbe978b502f0d7</td>\n",
       "      <td>2020-07-08 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>634</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Eagle Point</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-07-23 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>721</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c4640a78e89b6b196e0f0b8b91cbe978b502f0d7</td>\n",
       "      <td>2020-07-01 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>624</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Eagle Point</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382493</th>\n",
       "      <td>a84fd14a4996be166c9b90851342465523897e91</td>\n",
       "      <td>2020-07-06 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382494</th>\n",
       "      <td>a84fd14a4996be166c9b90851342465523897e91</td>\n",
       "      <td>2020-07-12 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382495</th>\n",
       "      <td>a84fd14a4996be166c9b90851342465523897e91</td>\n",
       "      <td>2020-07-25 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382496</th>\n",
       "      <td>a84fd14a4996be166c9b90851342465523897e91</td>\n",
       "      <td>2020-07-24 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382497</th>\n",
       "      <td>a84fd14a4996be166c9b90851342465523897e91</td>\n",
       "      <td>2020-07-08 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>116</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>356858 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c4640a78e89b6b196e0f0b8b91cbe978b502f0d7  2020-07-15 18:30:00 UTC   \n",
       "1       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2020-07-20 14:05:00 UTC   \n",
       "2       c4640a78e89b6b196e0f0b8b91cbe978b502f0d7  2020-07-08 17:55:00 UTC   \n",
       "3       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-07-23 19:50:00 UTC   \n",
       "4       c4640a78e89b6b196e0f0b8b91cbe978b502f0d7  2020-07-01 18:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "382493  a84fd14a4996be166c9b90851342465523897e91  2020-07-06 17:30:00 UTC   \n",
       "382494  a84fd14a4996be166c9b90851342465523897e91  2020-07-12 17:25:00 UTC   \n",
       "382495  a84fd14a4996be166c9b90851342465523897e91  2020-07-25 19:55:00 UTC   \n",
       "382496  a84fd14a4996be166c9b90851342465523897e91  2020-07-24 16:20:00 UTC   \n",
       "382497  a84fd14a4996be166c9b90851342465523897e91  2020-07-08 17:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               628                      620   \n",
       "1          auto          hold               747                      761   \n",
       "2          cool          auto               634                      620   \n",
       "3          cool          hold               722                      721   \n",
       "4          cool          auto               624                      620   \n",
       "...         ...           ...               ...                      ...   \n",
       "382493     heat          hold               698                      670   \n",
       "382494     heat          hold               697                      670   \n",
       "382495     heat          hold               691                      670   \n",
       "382496     heat          hold               690                      670   \n",
       "382497     heat          hold               695                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           620            OR    Eagle Point   \n",
       "1                           635            OR       PORTLAND   \n",
       "2                           620            OR    Eagle Point   \n",
       "3                           702            OR  Central point   \n",
       "4                           620            OR    Eagle Point   \n",
       "...                         ...           ...            ...   \n",
       "382493                      670            OR       Portland   \n",
       "382494                      670            OR       Portland   \n",
       "382495                      670            OR       Portland   \n",
       "382496                      670            OR       Portland   \n",
       "382497                      670            OR       Portland   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        16             False         False            False   \n",
       "1                         7             False         False             True   \n",
       "2                        16             False         False            False   \n",
       "3                        55             False         False            False   \n",
       "4                        16             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "382493                  116             False         False            False   \n",
       "382494                  116             False         False            False   \n",
       "382495                  116             False         False            False   \n",
       "382496                  116             False         False            False   \n",
       "382497                  116             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "382493                       Gas  \n",
       "382494                       Gas  \n",
       "382495                       Gas  \n",
       "382496                       Gas  \n",
       "382497                       Gas  \n",
       "\n",
       "[356858 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/OR/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/OR-day/2021-jul-day-OR.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2021-07-28 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>df98dfa78799eecb3c6e4ec08abd443231822f1b</td>\n",
       "      <td>2021-07-28 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>755</td>\n",
       "      <td>753</td>\n",
       "      <td>OR</td>\n",
       "      <td>Silverton</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2021-07-24 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>b9218cc02444b2c134490e36cc878268181619ce</td>\n",
       "      <td>2021-07-26 19:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>625</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b</td>\n",
       "      <td>2021-07-28 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>OR</td>\n",
       "      <td>Newberg</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199983</th>\n",
       "      <td>fa59ddbc4f617580e56d48d932fead97b7d8c83d</td>\n",
       "      <td>2021-07-27 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hillsboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199984</th>\n",
       "      <td>fa59ddbc4f617580e56d48d932fead97b7d8c83d</td>\n",
       "      <td>2021-07-27 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hillsboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199985</th>\n",
       "      <td>fa59ddbc4f617580e56d48d932fead97b7d8c83d</td>\n",
       "      <td>2021-07-28 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hillsboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199986</th>\n",
       "      <td>fa59ddbc4f617580e56d48d932fead97b7d8c83d</td>\n",
       "      <td>2021-07-05 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hillsboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199987</th>\n",
       "      <td>fa59ddbc4f617580e56d48d932fead97b7d8c83d</td>\n",
       "      <td>2021-07-05 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hillsboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>183306 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2021-07-28 18:20:00 UTC   \n",
       "1       df98dfa78799eecb3c6e4ec08abd443231822f1b  2021-07-28 16:45:00 UTC   \n",
       "2       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2021-07-24 18:25:00 UTC   \n",
       "3       b9218cc02444b2c134490e36cc878268181619ce  2021-07-26 19:35:00 UTC   \n",
       "4       fbd0793fdbaad09b9358ba988d5d27c2a4ea9c0b  2021-07-28 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "199983  fa59ddbc4f617580e56d48d932fead97b7d8c83d  2021-07-27 14:50:00 UTC   \n",
       "199984  fa59ddbc4f617580e56d48d932fead97b7d8c83d  2021-07-27 16:15:00 UTC   \n",
       "199985  fa59ddbc4f617580e56d48d932fead97b7d8c83d  2021-07-28 15:30:00 UTC   \n",
       "199986  fa59ddbc4f617580e56d48d932fead97b7d8c83d  2021-07-05 19:25:00 UTC   \n",
       "199987  fa59ddbc4f617580e56d48d932fead97b7d8c83d  2021-07-05 18:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               720                      734   \n",
       "1          cool          hold               757                      755   \n",
       "2          cool          hold               717                      716   \n",
       "3          auto          hold               675                      675   \n",
       "4          cool          hold               726                      734   \n",
       "...         ...           ...               ...                      ...   \n",
       "199983     cool          hold               738                      740   \n",
       "199984     cool          hold               737                      740   \n",
       "199985     cool          hold               728                      740   \n",
       "199986     cool          hold               727                      745   \n",
       "199987     cool          hold               723                      745   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           734            OR       Newberg   \n",
       "1                           753            OR     Silverton   \n",
       "2                           716            OR       Newberg   \n",
       "3                           625            OR  North Plains   \n",
       "4                           734            OR       Newberg   \n",
       "...                         ...           ...           ...   \n",
       "199983                      740            OR     Hillsboro   \n",
       "199984                      740            OR     Hillsboro   \n",
       "199985                      740            OR     Hillsboro   \n",
       "199986                      745            OR     Hillsboro   \n",
       "199987                      745            OR     Hillsboro   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        60              True         False             True   \n",
       "1                        90              True         False            False   \n",
       "2                        60              True         False             True   \n",
       "3                         0             False         False             True   \n",
       "4                        60              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "199983                   20             False         False            False   \n",
       "199984                   20             False         False            False   \n",
       "199985                   20             False         False            False   \n",
       "199986                   20             False         False            False   \n",
       "199987                   20             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                       Electric  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "199983                       Gas  \n",
       "199984                       Gas  \n",
       "199985                       Gas  \n",
       "199986                       Gas  \n",
       "199987                       Gas  \n",
       "\n",
       "[183306 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/OR/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006b3b1d03de005725492594b7733c3ef67d7dfe</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oregon City</td>\n",
       "      <td>724.785714</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>652.857143</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00b089459452edc92a0469d6ea9d74767924f3f5</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>697.667536</td>\n",
       "      <td>773.134289</td>\n",
       "      <td>772.559322</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>749.915663</td>\n",
       "      <td>760.337349</td>\n",
       "      <td>651.084337</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>760.350000</td>\n",
       "      <td>766.087500</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>04fac6711e3ad53990eebc8a7d198052deb2a9b1</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hood River</td>\n",
       "      <td>741.380952</td>\n",
       "      <td>740.751323</td>\n",
       "      <td>738.608466</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>fd6dd3503a19a2386a09f4b25e636ef7cfb34eb3</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Albany</td>\n",
       "      <td>724.153846</td>\n",
       "      <td>717.000000</td>\n",
       "      <td>717.000000</td>\n",
       "      <td>19.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>741.009009</td>\n",
       "      <td>745.033033</td>\n",
       "      <td>745.057057</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>701.576923</td>\n",
       "      <td>709.990385</td>\n",
       "      <td>709.980769</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>716.365385</td>\n",
       "      <td>736.346154</td>\n",
       "      <td>696.346154</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>716.164147</td>\n",
       "      <td>725.630670</td>\n",
       "      <td>725.557235</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3316 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    006b3b1d03de005725492594b7733c3ef67d7dfe   jul  2017     auto   \n",
       "1    00b089459452edc92a0469d6ea9d74767924f3f5   jul  2017     cool   \n",
       "2    03caad454a66062fc56b9128d6de2ca5ae4b7996   jul  2017     auto   \n",
       "3    03caad454a66062fc56b9128d6de2ca5ae4b7996   jul  2017     auto   \n",
       "4    04fac6711e3ad53990eebc8a7d198052deb2a9b1   jul  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "436  fd6dd3503a19a2386a09f4b25e636ef7cfb34eb3   jul  2021     cool   \n",
       "437  fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59   jul  2021     cool   \n",
       "438  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   jul  2021     cool   \n",
       "439  ff38b7d2e7a775a99129867043a265bf5cbb48b2   jul  2021     auto   \n",
       "440  ff38b7d2e7a775a99129867043a265bf5cbb48b2   jul  2021     cool   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto  Oregon City            724.785714   \n",
       "1            hold    Beaverton            697.667536   \n",
       "2            auto     Portland            749.915663   \n",
       "3            hold     Portland            760.350000   \n",
       "4            auto   Hood River            741.380952   \n",
       "..            ...          ...                   ...   \n",
       "436          hold       Albany            724.153846   \n",
       "437          hold     Roseburg            741.009009   \n",
       "438          hold    Beaverton            701.576923   \n",
       "439          hold      Gresham            716.365385   \n",
       "440          hold      Gresham            716.164147   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     720.000000                   652.857143   \n",
       "1                     773.134289                   772.559322   \n",
       "2                     760.337349                   651.084337   \n",
       "3                     766.087500                   640.000000   \n",
       "4                     740.751323                   738.608466   \n",
       "..                           ...                          ...   \n",
       "436                   717.000000                   717.000000   \n",
       "437                   745.033033                   745.057057   \n",
       "438                   709.990385                   709.980769   \n",
       "439                   736.346154                   696.346154   \n",
       "440                   725.630670                   725.557235   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   25.0              True         False            False  \n",
       "2                    5.0             False         False            False  \n",
       "3                    5.0             False         False            False  \n",
       "4                    0.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "436                 19.0             False         False            False  \n",
       "437                 20.0             False         False             True  \n",
       "438                  5.0             False         False            False  \n",
       "439                 20.0              True         False            False  \n",
       "440                 20.0              True         False            False  \n",
       "\n",
       "[3316 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/jul/\" + file)\n",
    "    OR_jul = pd.concat([OR_jul, df])\n",
    "    \n",
    "OR_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_jul.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/OR-day/2017-aug-day-OR.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2017-08-21 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>776</td>\n",
       "      <td>772</td>\n",
       "      <td>752</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2017-08-24 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>806</td>\n",
       "      <td>644</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2017-08-24 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>806</td>\n",
       "      <td>644</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ea5eb651c8e70d9cccebe4bb01a228a12f1079da</td>\n",
       "      <td>2017-08-06 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>758</td>\n",
       "      <td>758</td>\n",
       "      <td>OR</td>\n",
       "      <td>Hood River</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2017-08-21 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>772</td>\n",
       "      <td>752</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173493</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-08-22 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173494</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-08-22 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173495</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-08-20 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173496</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-08-10 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173497</th>\n",
       "      <td>a6c9df0eb0fa81661baef9d1a67507a9837690a5</td>\n",
       "      <td>2017-08-10 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>OR</td>\n",
       "      <td>forest grove</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>163445 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6c112fdef3cc591bfc328e7b5f040668e08165ca  2017-08-21 16:35:00 UTC   \n",
       "1       6c112fdef3cc591bfc328e7b5f040668e08165ca  2017-08-24 17:20:00 UTC   \n",
       "3       6c112fdef3cc591bfc328e7b5f040668e08165ca  2017-08-24 16:50:00 UTC   \n",
       "4       ea5eb651c8e70d9cccebe4bb01a228a12f1079da  2017-08-06 19:10:00 UTC   \n",
       "5       6c112fdef3cc591bfc328e7b5f040668e08165ca  2017-08-21 17:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "173493  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-08-22 18:20:00 UTC   \n",
       "173494  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-08-22 16:30:00 UTC   \n",
       "173495  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-08-20 19:45:00 UTC   \n",
       "173496  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-08-10 19:55:00 UTC   \n",
       "173497  a6c9df0eb0fa81661baef9d1a67507a9837690a5  2017-08-10 19:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               776                      772   \n",
       "1          auto          auto               783                      806   \n",
       "3          auto          auto               783                      806   \n",
       "4          cool          hold               745                      758   \n",
       "5          auto          hold               775                      772   \n",
       "...         ...           ...               ...                      ...   \n",
       "173493     cool          hold               717                      750   \n",
       "173494     cool          hold               713                      750   \n",
       "173495     cool          hold               695                      760   \n",
       "173496     cool          hold               734                      760   \n",
       "173497     cool          hold               732                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState          City  \\\n",
       "0                           752            OR  North Plains   \n",
       "1                           644            OR  North Plains   \n",
       "3                           644            OR  North Plains   \n",
       "4                           758            OR    Hood River   \n",
       "5                           752            OR  North Plains   \n",
       "...                         ...           ...           ...   \n",
       "173493                      750            OR  forest grove   \n",
       "173494                      750            OR  forest grove   \n",
       "173495                      760            OR  forest grove   \n",
       "173496                      760            OR  forest grove   \n",
       "173497                      760            OR  forest grove   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0              True         False            False   \n",
       "1                         0              True         False            False   \n",
       "3                         0              True         False            False   \n",
       "4                         0             False         False            False   \n",
       "5                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "173493                    0             False         False            False   \n",
       "173494                    0             False         False            False   \n",
       "173495                    0             False         False            False   \n",
       "173496                    0             False         False            False   \n",
       "173497                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "173493                       Gas  \n",
       "173494                       Gas  \n",
       "173495                       Gas  \n",
       "173496                       Gas  \n",
       "173497                       Gas  \n",
       "\n",
       "[163445 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/OR/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/OR-day/2018-aug-day-OR.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a41d1655a42dc8d83a638379cc627a6c3107fa58</td>\n",
       "      <td>2018-08-07 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sunriver</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d32546856f82383cac4ce8007c048507ca518c28</td>\n",
       "      <td>2018-08-19 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>OR</td>\n",
       "      <td>oak grove</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a41d1655a42dc8d83a638379cc627a6c3107fa58</td>\n",
       "      <td>2018-08-13 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>782</td>\n",
       "      <td>782</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sunriver</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8b80ed37aa10fdf227ad6fb3601047d49d0e049f</td>\n",
       "      <td>2018-08-01 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>767</td>\n",
       "      <td>708</td>\n",
       "      <td>OR</td>\n",
       "      <td>Stayton</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a41d1655a42dc8d83a638379cc627a6c3107fa58</td>\n",
       "      <td>2018-08-09 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sunriver</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322870</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-08-15 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322871</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-08-15 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322872</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-08-15 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322873</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-08-16 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322874</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-08-16 18:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>276693 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a41d1655a42dc8d83a638379cc627a6c3107fa58  2018-08-07 13:30:00 UTC   \n",
       "1       d32546856f82383cac4ce8007c048507ca518c28  2018-08-19 19:15:00 UTC   \n",
       "2       a41d1655a42dc8d83a638379cc627a6c3107fa58  2018-08-13 19:30:00 UTC   \n",
       "3       8b80ed37aa10fdf227ad6fb3601047d49d0e049f  2018-08-01 16:35:00 UTC   \n",
       "4       a41d1655a42dc8d83a638379cc627a6c3107fa58  2018-08-09 17:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "322870  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-08-15 16:05:00 UTC   \n",
       "322871  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-08-15 17:45:00 UTC   \n",
       "322872  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-08-15 18:15:00 UTC   \n",
       "322873  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-08-16 17:30:00 UTC   \n",
       "322874  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-08-16 18:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               719                      762   \n",
       "1          auto          hold               736                      755   \n",
       "2          cool          hold               707                      782   \n",
       "3          auto          hold               714                      767   \n",
       "4          cool          hold               689                      762   \n",
       "...         ...           ...               ...                      ...   \n",
       "322870     auto          hold               724                      760   \n",
       "322871     auto          hold               737                      760   \n",
       "322872     auto          hold               738                      760   \n",
       "322873     auto          hold               754                      760   \n",
       "322874     auto          hold               761                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           762            OR       Sunriver   \n",
       "1                           705            OR      oak grove   \n",
       "2                           782            OR       Sunriver   \n",
       "3                           708            OR        Stayton   \n",
       "4                           762            OR       Sunriver   \n",
       "...                         ...           ...            ...   \n",
       "322870                      710            OR  Klamath Falls   \n",
       "322871                      710            OR  Klamath Falls   \n",
       "322872                      710            OR  Klamath Falls   \n",
       "322873                      710            OR  Klamath Falls   \n",
       "322874                      710            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        30             False         False            False   \n",
       "1                        50             False         False            False   \n",
       "2                        30             False         False            False   \n",
       "3                        10              True         False             True   \n",
       "4                        30             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "322870                   29              True         False            False   \n",
       "322871                   29              True         False            False   \n",
       "322872                   29              True         False            False   \n",
       "322873                   29              True         False            False   \n",
       "322874                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "322870                       Gas  \n",
       "322871                       Gas  \n",
       "322872                       Gas  \n",
       "322873                       Gas  \n",
       "322874                       Gas  \n",
       "\n",
       "[276693 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/OR/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/OR-day/2019-aug-day-OR.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8ad786ec06c05d4ae243023be9b45144a11db7c4</td>\n",
       "      <td>2019-08-04 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>628</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Canyon City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2300fb4ee7123dad86ce6ac6318cc9156338d0c5</td>\n",
       "      <td>2019-08-06 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>810</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8ad786ec06c05d4ae243023be9b45144a11db7c4</td>\n",
       "      <td>2019-08-10 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>626</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>OR</td>\n",
       "      <td>Canyon City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2bc2a0b579a3741fc70ee49b2c34c5849de49bde</td>\n",
       "      <td>2019-08-02 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>692</td>\n",
       "      <td>642</td>\n",
       "      <td>OR</td>\n",
       "      <td>Stayton</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9bd7ce0e761eabf64adc070831d3afa3244f32e6</td>\n",
       "      <td>2019-08-16 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>768</td>\n",
       "      <td>775</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Shady Cove</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419484</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-08-13 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419485</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-08-30 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419486</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-08-30 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419487</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-08-13 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419488</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-08-30 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>384605 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8ad786ec06c05d4ae243023be9b45144a11db7c4  2019-08-04 18:00:00 UTC   \n",
       "1       2300fb4ee7123dad86ce6ac6318cc9156338d0c5  2019-08-06 16:05:00 UTC   \n",
       "2       8ad786ec06c05d4ae243023be9b45144a11db7c4  2019-08-10 15:35:00 UTC   \n",
       "3       2bc2a0b579a3741fc70ee49b2c34c5849de49bde  2019-08-02 17:20:00 UTC   \n",
       "4       9bd7ce0e761eabf64adc070831d3afa3244f32e6  2019-08-16 18:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "419484  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-08-13 18:15:00 UTC   \n",
       "419485  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-08-30 17:20:00 UTC   \n",
       "419486  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-08-30 19:50:00 UTC   \n",
       "419487  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-08-13 16:40:00 UTC   \n",
       "419488  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-08-30 19:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               628                      620   \n",
       "1          auto          hold               761                      810   \n",
       "2          cool          auto               626                      620   \n",
       "3          auto          hold               698                      692   \n",
       "4          auto          hold               768                      775   \n",
       "...         ...           ...               ...                      ...   \n",
       "419484     auto          hold               764                      760   \n",
       "419485     auto          hold               764                      760   \n",
       "419486     auto          hold               763                      760   \n",
       "419487     auto          hold               748                      760   \n",
       "419488     auto          hold               762                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           620            OR    Canyon City   \n",
       "1                           620            OR       PORTLAND   \n",
       "2                           620            OR    Canyon City   \n",
       "3                           642            OR        Stayton   \n",
       "4                           685            OR     Shady Cove   \n",
       "...                         ...           ...            ...   \n",
       "419484                      680            OR  Klamath Falls   \n",
       "419485                      680            OR  Klamath Falls   \n",
       "419486                      680            OR  Klamath Falls   \n",
       "419487                      680            OR  Klamath Falls   \n",
       "419488                      680            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False            False   \n",
       "1                        40              True         False             True   \n",
       "2                        10             False         False            False   \n",
       "3                        20             False         False             True   \n",
       "4                         9             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "419484                   29              True         False            False   \n",
       "419485                   29              True         False            False   \n",
       "419486                   29              True         False            False   \n",
       "419487                   29              True         False            False   \n",
       "419488                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "419484                       Gas  \n",
       "419485                       Gas  \n",
       "419486                       Gas  \n",
       "419487                       Gas  \n",
       "419488                       Gas  \n",
       "\n",
       "[384605 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/OR/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/OR-day/2020-aug-day-OR.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2020-08-24 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>752</td>\n",
       "      <td>635</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-08-05 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>731</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>74c4cb80267ba7e9bca86d95b6e46b07a55940c9</td>\n",
       "      <td>2020-08-14 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>OR</td>\n",
       "      <td>Springfield</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-08-30 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>721</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2020-08-24 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>752</td>\n",
       "      <td>635</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377433</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2020-08-21 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>735</td>\n",
       "      <td>665</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377434</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2020-08-30 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>735</td>\n",
       "      <td>665</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377435</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2020-08-30 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>735</td>\n",
       "      <td>665</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377436</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2020-08-30 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>745</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377437</th>\n",
       "      <td>e8cd29c205dfbd5aa461efb3de976b243a38835f</td>\n",
       "      <td>2020-08-30 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>745</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Gladstone</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>338108 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2020-08-24 16:50:00 UTC   \n",
       "3       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-08-05 17:50:00 UTC   \n",
       "4       74c4cb80267ba7e9bca86d95b6e46b07a55940c9  2020-08-14 19:55:00 UTC   \n",
       "6       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-08-30 19:55:00 UTC   \n",
       "7       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2020-08-24 15:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "377433  e8cd29c205dfbd5aa461efb3de976b243a38835f  2020-08-21 16:10:00 UTC   \n",
       "377434  e8cd29c205dfbd5aa461efb3de976b243a38835f  2020-08-30 19:55:00 UTC   \n",
       "377435  e8cd29c205dfbd5aa461efb3de976b243a38835f  2020-08-30 18:05:00 UTC   \n",
       "377436  e8cd29c205dfbd5aa461efb3de976b243a38835f  2020-08-30 16:00:00 UTC   \n",
       "377437  e8cd29c205dfbd5aa461efb3de976b243a38835f  2020-08-30 15:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          auto          hold               742                      752   \n",
       "3          cool          hold               737                      731   \n",
       "4          cool          hold               723                      722   \n",
       "6          cool          hold               722                      721   \n",
       "7          auto          hold               737                      752   \n",
       "...         ...           ...               ...                      ...   \n",
       "377433     auto          hold               734                      735   \n",
       "377434     auto          hold               685                      735   \n",
       "377435     auto          hold               678                      735   \n",
       "377436     auto          hold               685                      745   \n",
       "377437     auto          hold               674                      745   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           635            OR       PORTLAND   \n",
       "3                           702            OR  Central point   \n",
       "4                           722            OR    Springfield   \n",
       "6                           702            OR  Central point   \n",
       "7                           635            OR       PORTLAND   \n",
       "...                         ...           ...            ...   \n",
       "377433                      665            OR      Gladstone   \n",
       "377434                      665            OR      Gladstone   \n",
       "377435                      665            OR      Gladstone   \n",
       "377436                      685            OR      Gladstone   \n",
       "377437                      685            OR      Gladstone   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         7             False         False             True   \n",
       "3                        55             False         False            False   \n",
       "4                        19              True         False            False   \n",
       "6                        55             False         False            False   \n",
       "7                         7             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "377433                  100             False         False            False   \n",
       "377434                  100             False         False            False   \n",
       "377435                  100             False         False            False   \n",
       "377436                  100             False         False            False   \n",
       "377437                  100             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "7                       Electric  \n",
       "...                          ...  \n",
       "377433                       Gas  \n",
       "377434                       Gas  \n",
       "377435                       Gas  \n",
       "377436                       Gas  \n",
       "377437                       Gas  \n",
       "\n",
       "[338108 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/OR/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006b3b1d03de005725492594b7733c3ef67d7dfe</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oregon City</td>\n",
       "      <td>730.163462</td>\n",
       "      <td>742.846154</td>\n",
       "      <td>678.269231</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00b089459452edc92a0469d6ea9d74767924f3f5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>715.278226</td>\n",
       "      <td>774.619624</td>\n",
       "      <td>773.611559</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>052a8daf4686b3a7f4a807ef3239f57490eb40d2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Forest Grove</td>\n",
       "      <td>726.020833</td>\n",
       "      <td>735.000000</td>\n",
       "      <td>665.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>710.452656</td>\n",
       "      <td>737.496536</td>\n",
       "      <td>616.678984</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>717.414830</td>\n",
       "      <td>758.788243</td>\n",
       "      <td>663.389446</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>723.323529</td>\n",
       "      <td>720.264706</td>\n",
       "      <td>718.929412</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>822</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>733.481481</td>\n",
       "      <td>716.296296</td>\n",
       "      <td>716.296296</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>823</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>727.075301</td>\n",
       "      <td>744.662651</td>\n",
       "      <td>712.975904</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>824</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>732.364583</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>755.000000</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>825</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>723.331929</td>\n",
       "      <td>740.454928</td>\n",
       "      <td>735.400168</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2719 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    006b3b1d03de005725492594b7733c3ef67d7dfe   aug  2017     auto   \n",
       "1    00b089459452edc92a0469d6ea9d74767924f3f5   aug  2017     cool   \n",
       "2    052a8daf4686b3a7f4a807ef3239f57490eb40d2   aug  2017     auto   \n",
       "3    0624dc8fec23932f314f9e0f97a6c7fb059446b2   aug  2017     auto   \n",
       "4    0624dc8fec23932f314f9e0f97a6c7fb059446b2   aug  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "821  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   aug  2020     cool   \n",
       "822  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   aug  2020     cool   \n",
       "823  ff38b7d2e7a775a99129867043a265bf5cbb48b2   aug  2020     auto   \n",
       "824  ff38b7d2e7a775a99129867043a265bf5cbb48b2   aug  2020     cool   \n",
       "825  ff38b7d2e7a775a99129867043a265bf5cbb48b2   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0            auto   Oregon City            730.163462   \n",
       "1            hold     Beaverton            715.278226   \n",
       "2            hold  Forest Grove            726.020833   \n",
       "3            auto      Portland            710.452656   \n",
       "4            hold      Portland            717.414830   \n",
       "..            ...           ...                   ...   \n",
       "821          auto     Beaverton            723.323529   \n",
       "822          hold     Beaverton            733.481481   \n",
       "823          hold       Gresham            727.075301   \n",
       "824          auto       Gresham            732.364583   \n",
       "825          hold       Gresham            723.331929   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     742.846154                   678.269231   \n",
       "1                     774.619624                   773.611559   \n",
       "2                     735.000000                   665.000000   \n",
       "3                     737.496536                   616.678984   \n",
       "4                     758.788243                   663.389446   \n",
       "..                           ...                          ...   \n",
       "821                   720.264706                   718.929412   \n",
       "822                   716.296296                   716.296296   \n",
       "823                   744.662651                   712.975904   \n",
       "824                   740.000000                   755.000000   \n",
       "825                   740.454928                   735.400168   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   25.0              True         False            False  \n",
       "2                   10.0             False         False            False  \n",
       "3                    0.0             False         False            False  \n",
       "4                    0.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "821                  5.0             False         False            False  \n",
       "822                  5.0             False         False            False  \n",
       "823                 20.0              True         False            False  \n",
       "824                 20.0              True         False            False  \n",
       "825                 20.0              True         False            False  \n",
       "\n",
       "[2719 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/aug/\" + file)\n",
    "    OR_aug = pd.concat([OR_aug, df])\n",
    "    \n",
    "OR_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_aug.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/OR-day/2017-dec-day-OR.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cfeeb226f8297cddd35bf2408bdb7bd4071160d8</td>\n",
       "      <td>2017-12-13 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>652</td>\n",
       "      <td>799</td>\n",
       "      <td>651</td>\n",
       "      <td>OR</td>\n",
       "      <td>Dundee</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>97d0f5da7db1e75514930e823feaf6c84421dce2</td>\n",
       "      <td>2017-12-24 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>761</td>\n",
       "      <td>726</td>\n",
       "      <td>OR</td>\n",
       "      <td>Portland</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cd71171c504caf17e5e4786c93311b9ff08bdf66</td>\n",
       "      <td>2017-12-15 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>736</td>\n",
       "      <td>702</td>\n",
       "      <td>OR</td>\n",
       "      <td>Scappoose</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>dccaf065befac19f089838bd56e3d6ca78f374c6</td>\n",
       "      <td>2017-12-11 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>681</td>\n",
       "      <td>681</td>\n",
       "      <td>OR</td>\n",
       "      <td>corvallis</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1c927546884d866d0498c730035be2e4464967b2</td>\n",
       "      <td>2017-12-22 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>OR</td>\n",
       "      <td>Lincoln City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237622</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2017-12-02 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237623</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2017-12-02 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237624</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2017-12-12 15:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237625</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2017-12-02 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237626</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2017-12-02 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>223603 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       cfeeb226f8297cddd35bf2408bdb7bd4071160d8  2017-12-13 15:45:00 UTC   \n",
       "2       97d0f5da7db1e75514930e823feaf6c84421dce2  2017-12-24 17:00:00 UTC   \n",
       "4       cd71171c504caf17e5e4786c93311b9ff08bdf66  2017-12-15 19:05:00 UTC   \n",
       "10      dccaf065befac19f089838bd56e3d6ca78f374c6  2017-12-11 16:10:00 UTC   \n",
       "11      1c927546884d866d0498c730035be2e4464967b2  2017-12-22 19:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "237622  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2017-12-02 19:05:00 UTC   \n",
       "237623  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2017-12-02 19:25:00 UTC   \n",
       "237624  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2017-12-12 15:45:00 UTC   \n",
       "237625  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2017-12-02 19:20:00 UTC   \n",
       "237626  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2017-12-02 17:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               652                      799   \n",
       "2          heat          hold               737                      761   \n",
       "4          heat          auto               705                      736   \n",
       "10         heat          auto               675                      681   \n",
       "11         heat          auto               691                      687   \n",
       "...         ...           ...               ...                      ...   \n",
       "237622     heat          hold               709                      710   \n",
       "237623     heat          hold               704                      710   \n",
       "237624     heat          hold               698                      710   \n",
       "237625     heat          hold               706                      710   \n",
       "237626     heat          hold               685                      720   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           651            OR         Dundee   \n",
       "2                           726            OR      Portland    \n",
       "4                           702            OR      Scappoose   \n",
       "10                          681            OR      corvallis   \n",
       "11                          687            OR   Lincoln City   \n",
       "...                         ...           ...            ...   \n",
       "237622                      710            OR  Klamath Falls   \n",
       "237623                      710            OR  Klamath Falls   \n",
       "237624                      710            OR  Klamath Falls   \n",
       "237625                      710            OR  Klamath Falls   \n",
       "237626                      720            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        47             False         False            False   \n",
       "2                        20             False         False            False   \n",
       "4                        27             False         False            False   \n",
       "10                        5             False         False            False   \n",
       "11                        0             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "237622                   29              True         False            False   \n",
       "237623                   29              True         False            False   \n",
       "237624                   29              True         False            False   \n",
       "237625                   29              True         False            False   \n",
       "237626                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "10                           Gas  \n",
       "11                           Gas  \n",
       "...                          ...  \n",
       "237622                       Gas  \n",
       "237623                       Gas  \n",
       "237624                       Gas  \n",
       "237625                       Gas  \n",
       "237626                       Gas  \n",
       "\n",
       "[223603 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/OR/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/OR-day/2018-dec-day-OR.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7474bc9ac147902665559a735c4b7820195e2d9d</td>\n",
       "      <td>2018-12-08 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>687</td>\n",
       "      <td>716</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>King City</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6</td>\n",
       "      <td>2018-12-24 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>778</td>\n",
       "      <td>840</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elkton</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6c112fdef3cc591bfc328e7b5f040668e08165ca</td>\n",
       "      <td>2018-12-12 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>761</td>\n",
       "      <td>761</td>\n",
       "      <td>OR</td>\n",
       "      <td>North Plains</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dc462408624c5f44e41d050ae70ca0bd895f78d0</td>\n",
       "      <td>2018-12-24 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>614</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>OR</td>\n",
       "      <td>Redmond</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d667da3526d79b5b1165e06da5ea6013ccfb3f13</td>\n",
       "      <td>2018-12-06 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>OR</td>\n",
       "      <td>Saint Helens</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350185</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-12-02 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350186</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2018-12-01 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350187</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2018-12-08 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350188</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2018-12-09 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>350189</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2018-12-08 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>725</td>\n",
       "      <td>685</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>323113 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7474bc9ac147902665559a735c4b7820195e2d9d  2018-12-08 13:10:00 UTC   \n",
       "1       d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6  2018-12-24 19:50:00 UTC   \n",
       "2       6c112fdef3cc591bfc328e7b5f040668e08165ca  2018-12-12 19:20:00 UTC   \n",
       "3       dc462408624c5f44e41d050ae70ca0bd895f78d0  2018-12-24 15:00:00 UTC   \n",
       "5       d667da3526d79b5b1165e06da5ea6013ccfb3f13  2018-12-06 19:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "350185  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-12-02 07:15:00 UTC   \n",
       "350186  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2018-12-01 07:15:00 UTC   \n",
       "350187  69bce0cead73bc1724ae942dad09e3393da90e97  2018-12-08 17:40:00 UTC   \n",
       "350188  69bce0cead73bc1724ae942dad09e3393da90e97  2018-12-09 17:10:00 UTC   \n",
       "350189  69bce0cead73bc1724ae942dad09e3393da90e97  2018-12-08 17:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               687                      716   \n",
       "1          auto          auto               778                      840   \n",
       "2          heat          hold               755                      761   \n",
       "3          heat          auto               614                      630   \n",
       "5          heat          hold               687                      722   \n",
       "...         ...           ...               ...                      ...   \n",
       "350185     heat          hold               705                      710   \n",
       "350186     heat          hold               716                      720   \n",
       "350187     heat          hold               685                      725   \n",
       "350188     heat          hold               685                      725   \n",
       "350189     heat          hold               684                      725   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           685            OR      King City   \n",
       "1                           650            OR         Elkton   \n",
       "2                           761            OR   North Plains   \n",
       "3                           630            OR        Redmond   \n",
       "5                           722            OR   Saint Helens   \n",
       "...                         ...           ...            ...   \n",
       "350185                      710            OR  Klamath Falls   \n",
       "350186                      720            OR  Klamath Falls   \n",
       "350187                      685            OR  Klamath Falls   \n",
       "350188                      685            OR  Klamath Falls   \n",
       "350189                      685            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "1                        40              True         False             True   \n",
       "2                         0              True         False            False   \n",
       "3                         0             False         False            False   \n",
       "5                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "350185                   29              True         False            False   \n",
       "350186                   29              True         False            False   \n",
       "350187                   40             False         False            False   \n",
       "350188                   40             False         False            False   \n",
       "350189                   40             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "350185                       Gas  \n",
       "350186                       Gas  \n",
       "350187                       Gas  \n",
       "350188                       Gas  \n",
       "350189                       Gas  \n",
       "\n",
       "[323113 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/OR/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/OR-day/2019-dec-day-OR.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>876b9dfe5bf0540d52e634786e681978524b51a1</td>\n",
       "      <td>2019-12-31 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>714</td>\n",
       "      <td>670</td>\n",
       "      <td>OR</td>\n",
       "      <td>McMinnville</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6</td>\n",
       "      <td>2019-12-14 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>840</td>\n",
       "      <td>650</td>\n",
       "      <td>OR</td>\n",
       "      <td>Elkton</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-12-12 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3ba502441ef84b6c417047c5cbfb88e7fa1776c2</td>\n",
       "      <td>2019-12-03 14:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>765</td>\n",
       "      <td>705</td>\n",
       "      <td>OR</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>53a8afb9c2869da7d84a186b864ea3c571c2646c</td>\n",
       "      <td>2019-12-18 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>679</td>\n",
       "      <td>679</td>\n",
       "      <td>OR</td>\n",
       "      <td>Sheridan</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424212</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-12-25 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424213</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-12-04 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424214</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-12-25 19:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424215</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-12-25 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424216</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2019-12-25 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>760</td>\n",
       "      <td>692</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>403469 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "1       876b9dfe5bf0540d52e634786e681978524b51a1  2019-12-31 15:30:00 UTC   \n",
       "4       d0fd4191a6b97277a0e5ad601efac94c0b9b3bb6  2019-12-14 19:50:00 UTC   \n",
       "5       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-12-12 13:45:00 UTC   \n",
       "6       3ba502441ef84b6c417047c5cbfb88e7fa1776c2  2019-12-03 14:30:00 UTC   \n",
       "7       53a8afb9c2869da7d84a186b864ea3c571c2646c  2019-12-18 07:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "424212  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-12-25 07:05:00 UTC   \n",
       "424213  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-12-04 07:05:00 UTC   \n",
       "424214  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-12-25 19:55:00 UTC   \n",
       "424215  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-12-25 19:25:00 UTC   \n",
       "424216  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2019-12-25 07:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1          heat          hold               661                      714   \n",
       "4          auto          hold               652                      840   \n",
       "5          auto          hold               693                      755   \n",
       "6          auto          hold               698                      765   \n",
       "7          heat          hold               670                      679   \n",
       "...         ...           ...               ...                      ...   \n",
       "424212     heat          hold               712                      720   \n",
       "424213     heat          hold               713                      720   \n",
       "424214     heat          hold               717                      720   \n",
       "424215     heat          hold               719                      720   \n",
       "424216     heat          hold               715                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "1                           670            OR    McMinnville   \n",
       "4                           650            OR         Elkton   \n",
       "5                           705            OR       Roseburg   \n",
       "6                           705            OR       Roseburg   \n",
       "7                           679            OR       Sheridan   \n",
       "...                         ...           ...            ...   \n",
       "424212                      720            OR  Klamath Falls   \n",
       "424213                      720            OR  Klamath Falls   \n",
       "424214                      720            OR  Klamath Falls   \n",
       "424215                      720            OR  Klamath Falls   \n",
       "424216                      692            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                        10              True         False             True   \n",
       "4                        40              True         False             True   \n",
       "5                        90             False         False            False   \n",
       "6                        90             False         False            False   \n",
       "7                        10             False         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "424212                   29              True         False            False   \n",
       "424213                   29              True         False            False   \n",
       "424214                   29              True         False            False   \n",
       "424215                   29              True         False            False   \n",
       "424216                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "1                       Electric  \n",
       "4                       Electric  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "7                       Electric  \n",
       "...                          ...  \n",
       "424212                       Gas  \n",
       "424213                       Gas  \n",
       "424214                       Gas  \n",
       "424215                       Gas  \n",
       "424216                       Gas  \n",
       "\n",
       "[403469 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/OR/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/OR-day/2020-dec-day-OR.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-12-18 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>674</td>\n",
       "      <td>681</td>\n",
       "      <td>681</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0f6950236a3fc5e05032d841d5e3dd47e144ef02</td>\n",
       "      <td>2020-12-24 18:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>788</td>\n",
       "      <td>698</td>\n",
       "      <td>OR</td>\n",
       "      <td>PORTLAND</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>876b9dfe5bf0540d52e634786e681978524b51a1</td>\n",
       "      <td>2020-12-12 16:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>628</td>\n",
       "      <td>714</td>\n",
       "      <td>630</td>\n",
       "      <td>OR</td>\n",
       "      <td>McMinnville</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>114702ca6a8da3f42947dd25a6d56a0e60c41541</td>\n",
       "      <td>2020-12-09 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>671</td>\n",
       "      <td>671</td>\n",
       "      <td>OR</td>\n",
       "      <td>Central point</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>876b9dfe5bf0540d52e634786e681978524b51a1</td>\n",
       "      <td>2020-12-10 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>714</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>McMinnville</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383056</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-12-08 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383057</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-12-13 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383058</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2020-12-27 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>725</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383059</th>\n",
       "      <td>69bce0cead73bc1724ae942dad09e3393da90e97</td>\n",
       "      <td>2020-12-27 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>725</td>\n",
       "      <td>690</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383060</th>\n",
       "      <td>7fa4b9bccf19f52cac8c9a629651366fd8570a89</td>\n",
       "      <td>2020-12-09 07:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>730</td>\n",
       "      <td>706</td>\n",
       "      <td>OR</td>\n",
       "      <td>Klamath Falls</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>367548 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-12-18 17:45:00 UTC   \n",
       "1       0f6950236a3fc5e05032d841d5e3dd47e144ef02  2020-12-24 18:15:00 UTC   \n",
       "2       876b9dfe5bf0540d52e634786e681978524b51a1  2020-12-12 16:20:00 UTC   \n",
       "3       114702ca6a8da3f42947dd25a6d56a0e60c41541  2020-12-09 19:05:00 UTC   \n",
       "4       876b9dfe5bf0540d52e634786e681978524b51a1  2020-12-10 16:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "383056  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-12-08 18:45:00 UTC   \n",
       "383057  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-12-13 19:35:00 UTC   \n",
       "383058  69bce0cead73bc1724ae942dad09e3393da90e97  2020-12-27 18:30:00 UTC   \n",
       "383059  69bce0cead73bc1724ae942dad09e3393da90e97  2020-12-27 18:40:00 UTC   \n",
       "383060  7fa4b9bccf19f52cac8c9a629651366fd8570a89  2020-12-09 07:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               674                      681   \n",
       "1          auto          hold               698                      788   \n",
       "2          heat          auto               628                      714   \n",
       "3          heat          hold               666                      671   \n",
       "4          heat          hold               645                      714   \n",
       "...         ...           ...               ...                      ...   \n",
       "383056     heat          hold               716                      720   \n",
       "383057     heat          hold               699                      720   \n",
       "383058     heat          hold               683                      725   \n",
       "383059     heat          hold               690                      725   \n",
       "383060     heat          hold               715                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           681            OR  Central point   \n",
       "1                           698            OR       PORTLAND   \n",
       "2                           630            OR    McMinnville   \n",
       "3                           671            OR  Central point   \n",
       "4                           690            OR    McMinnville   \n",
       "...                         ...           ...            ...   \n",
       "383056                      720            OR  Klamath Falls   \n",
       "383057                      720            OR  Klamath Falls   \n",
       "383058                      690            OR  Klamath Falls   \n",
       "383059                      690            OR  Klamath Falls   \n",
       "383060                      706            OR  Klamath Falls   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        55             False         False            False   \n",
       "1                         7             False         False             True   \n",
       "2                        10              True         False             True   \n",
       "3                        55             False         False            False   \n",
       "4                        10              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "383056                   29              True         False            False   \n",
       "383057                   29              True         False            False   \n",
       "383058                   40             False         False            False   \n",
       "383059                   40             False         False            False   \n",
       "383060                   29              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                       Electric  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "383056                       Gas  \n",
       "383057                       Gas  \n",
       "383058                       Gas  \n",
       "383059                       Gas  \n",
       "383060                       Gas  \n",
       "\n",
       "[367548 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/OR/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/OR/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006b3b1d03de005725492594b7733c3ef67d7dfe</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oregon City</td>\n",
       "      <td>716.652778</td>\n",
       "      <td>757.222222</td>\n",
       "      <td>723.194444</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00bc4f328673432b1fd2fff05c6a88a846f61605</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>656.181818</td>\n",
       "      <td>669.090909</td>\n",
       "      <td>668.590909</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00bc4f328673432b1fd2fff05c6a88a846f61605</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>646.182160</td>\n",
       "      <td>658.383099</td>\n",
       "      <td>652.127700</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>691.159533</td>\n",
       "      <td>691.027237</td>\n",
       "      <td>690.513619</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>03caad454a66062fc56b9128d6de2ca5ae4b7996</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>688.194444</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>908</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>White City</td>\n",
       "      <td>704.694444</td>\n",
       "      <td>745.277778</td>\n",
       "      <td>693.888889</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>909</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>White City</td>\n",
       "      <td>714.272727</td>\n",
       "      <td>698.181818</td>\n",
       "      <td>698.181818</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>910</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>710.416667</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>719.750000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>911</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>701.941176</td>\n",
       "      <td>708.358289</td>\n",
       "      <td>708.181818</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>705.829787</td>\n",
       "      <td>708.744681</td>\n",
       "      <td>707.765957</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3530 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    006b3b1d03de005725492594b7733c3ef67d7dfe   dec  2017     heat   \n",
       "1    00bc4f328673432b1fd2fff05c6a88a846f61605   dec  2017     heat   \n",
       "2    00bc4f328673432b1fd2fff05c6a88a846f61605   dec  2017     heat   \n",
       "3    03caad454a66062fc56b9128d6de2ca5ae4b7996   dec  2017     heat   \n",
       "4    03caad454a66062fc56b9128d6de2ca5ae4b7996   dec  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "908  fe918247d6bdbd347613dd68f15c85893bb512da   dec  2020     heat   \n",
       "909  fe918247d6bdbd347613dd68f15c85893bb512da   dec  2020     heat   \n",
       "910  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   dec  2020     heat   \n",
       "911  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   dec  2020     heat   \n",
       "912  ff38b7d2e7a775a99129867043a265bf5cbb48b2   dec  2020     heat   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto  Oregon City            716.652778   \n",
       "1            auto     Portland            656.181818   \n",
       "2            hold     Portland            646.182160   \n",
       "3            auto     Portland            691.159533   \n",
       "4            hold     Portland            688.194444   \n",
       "..            ...          ...                   ...   \n",
       "908          auto   White City            704.694444   \n",
       "909          hold   White City            714.272727   \n",
       "910          auto    Beaverton            710.416667   \n",
       "911          hold    Beaverton            701.941176   \n",
       "912          hold      Gresham            705.829787   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     757.222222                   723.194444   \n",
       "1                     669.090909                   668.590909   \n",
       "2                     658.383099                   652.127700   \n",
       "3                     691.027237                   690.513619   \n",
       "4                     690.000000                   690.000000   \n",
       "..                           ...                          ...   \n",
       "908                   745.277778                   693.888889   \n",
       "909                   698.181818                   698.181818   \n",
       "910                   740.000000                   719.750000   \n",
       "911                   708.358289                   708.181818   \n",
       "912                   708.744681                   707.765957   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   20.0             False         False            False  \n",
       "1                   20.0             False         False            False  \n",
       "2                   20.0             False         False            False  \n",
       "3                    5.0             False         False            False  \n",
       "4                    5.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "908                 57.0             False         False            False  \n",
       "909                 57.0             False         False            False  \n",
       "910                  5.0             False         False            False  \n",
       "911                  5.0             False         False            False  \n",
       "912                 20.0              True         False            False  \n",
       "\n",
       "[3530 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/OR/dec/\" + file)\n",
    "    OR_dec = pd.concat([OR_dec, df])\n",
    "    \n",
    "OR_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_dec.to_csv(\"Scraper_Output/State_Month_Day/OR/OR_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/OR/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>006b3b1d03de005725492594b7733c3ef67d7dfe</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oregon City</td>\n",
       "      <td>730.163462</td>\n",
       "      <td>742.846154</td>\n",
       "      <td>678.269231</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00b089459452edc92a0469d6ea9d74767924f3f5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>715.278226</td>\n",
       "      <td>774.619624</td>\n",
       "      <td>773.611559</td>\n",
       "      <td>25.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>052a8daf4686b3a7f4a807ef3239f57490eb40d2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Forest Grove</td>\n",
       "      <td>726.020833</td>\n",
       "      <td>735.000000</td>\n",
       "      <td>665.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Portland</td>\n",
       "      <td>710.452656</td>\n",
       "      <td>737.496536</td>\n",
       "      <td>616.678984</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0624dc8fec23932f314f9e0f97a6c7fb059446b2</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Portland</td>\n",
       "      <td>717.414830</td>\n",
       "      <td>758.788243</td>\n",
       "      <td>663.389446</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3876</th>\n",
       "      <td>fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Roseburg</td>\n",
       "      <td>761.627907</td>\n",
       "      <td>756.250000</td>\n",
       "      <td>756.366279</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3877</th>\n",
       "      <td>fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Albany</td>\n",
       "      <td>670.222222</td>\n",
       "      <td>773.777778</td>\n",
       "      <td>773.777778</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3878</th>\n",
       "      <td>fe918247d6bdbd347613dd68f15c85893bb512da</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>White City</td>\n",
       "      <td>691.432836</td>\n",
       "      <td>679.552239</td>\n",
       "      <td>679.059701</td>\n",
       "      <td>57.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3879</th>\n",
       "      <td>ff17c244fe42179fd802221bc9b6c177b3ad7d3e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Beaverton</td>\n",
       "      <td>712.991935</td>\n",
       "      <td>708.838710</td>\n",
       "      <td>708.766129</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3880</th>\n",
       "      <td>ff38b7d2e7a775a99129867043a265bf5cbb48b2</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Gresham</td>\n",
       "      <td>712.201278</td>\n",
       "      <td>723.092652</td>\n",
       "      <td>723.035144</td>\n",
       "      <td>20.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20056 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     006b3b1d03de005725492594b7733c3ef67d7dfe   aug  2017     auto   \n",
       "1     00b089459452edc92a0469d6ea9d74767924f3f5   aug  2017     cool   \n",
       "2     052a8daf4686b3a7f4a807ef3239f57490eb40d2   aug  2017     auto   \n",
       "3     0624dc8fec23932f314f9e0f97a6c7fb059446b2   aug  2017     auto   \n",
       "4     0624dc8fec23932f314f9e0f97a6c7fb059446b2   aug  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3876  fd8e322cdf93026aa75f2e65d3a8ffe6a4cdeb59   jun  2021     cool   \n",
       "3877  fe1baa9f35e9f8ec94a0cd7e1e93b43435a7ee8a   jun  2021     cool   \n",
       "3878  fe918247d6bdbd347613dd68f15c85893bb512da   jun  2021     cool   \n",
       "3879  ff17c244fe42179fd802221bc9b6c177b3ad7d3e   jun  2021     cool   \n",
       "3880  ff38b7d2e7a775a99129867043a265bf5cbb48b2   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto   Oregon City            730.163462   \n",
       "1             hold     Beaverton            715.278226   \n",
       "2             hold  Forest Grove            726.020833   \n",
       "3             auto      Portland            710.452656   \n",
       "4             hold      Portland            717.414830   \n",
       "...            ...           ...                   ...   \n",
       "3876          hold      Roseburg            761.627907   \n",
       "3877          hold        Albany            670.222222   \n",
       "3878          hold    White City            691.432836   \n",
       "3879          hold     Beaverton            712.991935   \n",
       "3880          hold       Gresham            712.201278   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      742.846154                   678.269231   \n",
       "1                      774.619624                   773.611559   \n",
       "2                      735.000000                   665.000000   \n",
       "3                      737.496536                   616.678984   \n",
       "4                      758.788243                   663.389446   \n",
       "...                           ...                          ...   \n",
       "3876                   756.250000                   756.366279   \n",
       "3877                   773.777778                   773.777778   \n",
       "3878                   679.552239                   679.059701   \n",
       "3879                   708.838710                   708.766129   \n",
       "3880                   723.092652                   723.035144   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    20.0             False         False            False  \n",
       "1                    25.0              True         False            False  \n",
       "2                    10.0             False         False            False  \n",
       "3                     0.0             False         False            False  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "3876                 20.0             False         False             True  \n",
       "3877                120.0             False         False            False  \n",
       "3878                 57.0             False         False            False  \n",
       "3879                  5.0             False         False            False  \n",
       "3880                 20.0              True         False            False  \n",
       "\n",
       "[20056 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "OR_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/OR/\" + file)\n",
    "    OR_all = pd.concat([OR_all, df])\n",
    "    \n",
    "OR_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "OR_all.to_csv(\"Scraper_Output/State_Month_Day/OR_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['OR']\n",
      "Unique jan_2018: ['OR']\n",
      "Unique jan_2019: ['OR']\n",
      "Unique jan_2020: ['OR']\n",
      "Unique jan_2021: ['OR']\n",
      "Unique feb_2017: ['OR']\n",
      "Unique feb_2018: ['OR']\n",
      "Unique feb_2019: ['OR']\n",
      "Unique feb_2020: ['OR']\n",
      "Unique feb_2021: ['OR']\n",
      "Unique jun_2017: ['OR']\n",
      "Unique jun_2018: ['OR']\n",
      "Unique jun_2019: ['OR']\n",
      "Unique jun_2020: ['OR']\n",
      "Unique jun_2021: ['OR']\n",
      "Unique jul_2017: ['OR']\n",
      "Unique jul_2018: ['OR']\n",
      "Unique jul_2019: ['OR']\n",
      "Unique jul_2020: ['OR']\n",
      "Unique jul_2021: ['OR']\n",
      "Unique aug_2017: ['OR']\n",
      "Unique aug_2018: ['OR']\n",
      "Unique aug_2019: ['OR']\n",
      "Unique aug_2020: ['OR']\n",
      "Unique dec_2017: ['OR']\n",
      "Unique dec_2018: ['OR']\n",
      "Unique dec_2019: ['OR']\n",
      "Unique dec_2020: ['OR']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mORe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}