DYD_KS-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/KS-day/2017-jan-day-KS.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-01-12 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-01-13 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-01-01 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-01-14 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-01-13 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95247</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-01-14 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>671</td>\n",
       "      <td>740</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95248</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-01-22 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>638</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95249</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-01-21 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>654</td>\n",
       "      <td>740</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95250</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-01-21 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>653</td>\n",
       "      <td>740</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95251</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-01-14 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>740</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93121 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-01-12 12:40:00 UTC   \n",
       "1      beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-01-13 14:00:00 UTC   \n",
       "2      beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-01-01 17:55:00 UTC   \n",
       "3      beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-01-14 18:35:00 UTC   \n",
       "4      beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-01-13 16:00:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "95247  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-01-14 19:30:00 UTC   \n",
       "95248  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-01-22 15:25:00 UTC   \n",
       "95249  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-01-21 16:25:00 UTC   \n",
       "95250  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-01-21 16:45:00 UTC   \n",
       "95251  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-01-14 19:55:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          hold               675                      680   \n",
       "1         heat          hold               678                      680   \n",
       "2         heat          hold               673                      670   \n",
       "3         heat          hold               687                      690   \n",
       "4         heat          hold               681                      680   \n",
       "...        ...           ...               ...                      ...   \n",
       "95247     auto          auto               671                      740   \n",
       "95248     auto          auto               638                      720   \n",
       "95249     auto          auto               654                      740   \n",
       "95250     auto          auto               653                      740   \n",
       "95251     auto          auto               689                      740   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                          680            KS  Eastborough   \n",
       "1                          680            KS  Eastborough   \n",
       "2                          670            KS  Eastborough   \n",
       "3                          690            KS  Eastborough   \n",
       "4                          680            KS  Eastborough   \n",
       "...                        ...           ...          ...   \n",
       "95247                      690            KS     Lawrence   \n",
       "95248                      640            KS     Lawrence   \n",
       "95249                      640            KS     Lawrence   \n",
       "95250                      640            KS     Lawrence   \n",
       "95251                      690            KS     Lawrence   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       79              True         False            False   \n",
       "1                       79              True         False            False   \n",
       "2                       79              True         False            False   \n",
       "3                       79              True         False            False   \n",
       "4                       79              True         False            False   \n",
       "...                    ...               ...           ...              ...   \n",
       "95247                  120             False         False            False   \n",
       "95248                  120             False         False            False   \n",
       "95249                  120             False         False            False   \n",
       "95250                  120             False         False            False   \n",
       "95251                  120             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "95247                       Gas  \n",
       "95248                       Gas  \n",
       "95249                       Gas  \n",
       "95250                       Gas  \n",
       "95251                       Gas  \n",
       "\n",
       "[93121 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Fairway</th>\n",
       "      <td>669.450980</td>\n",
       "      <td>820.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">01a0d7a563a1755d37bb856a01a56e452deb2010</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>688.583048</td>\n",
       "      <td>799.958904</td>\n",
       "      <td>690.002568</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cool</th>\n",
       "      <th>hold</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>652.261905</td>\n",
       "      <td>713.261905</td>\n",
       "      <td>713.142857</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>697.493721</td>\n",
       "      <td>698.607535</td>\n",
       "      <td>698.607535</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0448b121ebca96144ab5e5feb5ab88942e98de42</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auxHeatOnly</th>\n",
       "      <th>auto</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>691.707317</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fdadb8143c9f8d15c6bfbffab7628b015c5db3cb</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>627.546154</td>\n",
       "      <td>653.153846</td>\n",
       "      <td>636.669231</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Olathe</th>\n",
       "      <td>679.081633</td>\n",
       "      <td>680.489796</td>\n",
       "      <td>680.489796</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fe779904e8eede7395997bd2762cd654983d363a</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Andover</th>\n",
       "      <td>671.520000</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>620.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fec0465e4c757d26b069a20df5b2451426614367</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Overland Park</th>\n",
       "      <td>682.363636</td>\n",
       "      <td>765.454545</td>\n",
       "      <td>683.636364</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Overland Park</th>\n",
       "      <td>670.975000</td>\n",
       "      <td>675.550000</td>\n",
       "      <td>675.100000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>237 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                             Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                  \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                  669.450980   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                   688.583048   \n",
       "                                                    cool        hold          Olathe                   652.261905   \n",
       "                                                    heat        hold          Olathe                   697.493721   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                   691.707317   \n",
       "...                                                                                                           ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                   627.546154   \n",
       "                                                                hold          Olathe                   679.081633   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                  671.520000   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park            682.363636   \n",
       "                                                                hold          Overland Park            670.975000   \n",
       "\n",
       "                                                                                             TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                         \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                         820.000000   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                          799.958904   \n",
       "                                                    cool        hold          Olathe                          713.261905   \n",
       "                                                    heat        hold          Olathe                          698.607535   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                          780.000000   \n",
       "...                                                                                                                  ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                          653.153846   \n",
       "                                                                hold          Olathe                          680.489796   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                         650.000000   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park                   765.454545   \n",
       "                                                                hold          Overland Park                   675.550000   \n",
       "\n",
       "                                                                                             TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                         \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                         640.000000   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                          690.002568   \n",
       "                                                    cool        hold          Olathe                          713.142857   \n",
       "                                                    heat        hold          Olathe                          698.607535   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                          690.000000   \n",
       "...                                                                                                                  ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                          636.669231   \n",
       "                                                                hold          Olathe                          680.489796   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                         620.000000   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park                   683.636364   \n",
       "                                                                hold          Overland Park                   675.100000   \n",
       "\n",
       "                                                                                             Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                                 \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                       80.0   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                        30.0   \n",
       "                                                    cool        hold          Olathe                        30.0   \n",
       "                                                    heat        hold          Olathe                        30.0   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                        10.0   \n",
       "...                                                                                                          ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                        35.0   \n",
       "                                                                hold          Olathe                        35.0   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                        0.0   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park                 25.0   \n",
       "                                                                hold          Overland Park                 25.0   \n",
       "\n",
       "                                                                                             allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                              \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                   False   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                    False   \n",
       "                                                    cool        hold          Olathe                    False   \n",
       "                                                    heat        hold          Olathe                    False   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                    False   \n",
       "...                                                                                                       ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                    False   \n",
       "                                                                hold          Olathe                    False   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                   False   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park             False   \n",
       "                                                                hold          Overland Park             False   \n",
       "\n",
       "                                                                                             Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                          \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway               False   \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                False   \n",
       "                                                    cool        hold          Olathe                False   \n",
       "                                                    heat        hold          Olathe                False   \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                False   \n",
       "...                                                                                                   ...   \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                False   \n",
       "                                                                hold          Olathe                False   \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover               False   \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park         False   \n",
       "                                                                hold          Overland Park         False   \n",
       "\n",
       "                                                                                             Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode    CalendarEvent City                            \n",
       "00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e Jan   2017 auto        auto          Fairway                  False  \n",
       "01a0d7a563a1755d37bb856a01a56e452deb2010 Jan   2017 auto        hold          Olathe                   False  \n",
       "                                                    cool        hold          Olathe                   False  \n",
       "                                                    heat        hold          Olathe                   False  \n",
       "0448b121ebca96144ab5e5feb5ab88942e98de42 Jan   2017 auxHeatOnly auto          Olathe                   False  \n",
       "...                                                                                                      ...  \n",
       "fdadb8143c9f8d15c6bfbffab7628b015c5db3cb Jan   2017 heat        auto          Olathe                   False  \n",
       "                                                                hold          Olathe                   False  \n",
       "fe779904e8eede7395997bd2762cd654983d363a Jan   2017 heat        auto          Andover                  False  \n",
       "fec0465e4c757d26b069a20df5b2451426614367 Jan   2017 heat        auto          Overland Park            False  \n",
       "                                                                hold          Overland Park            False  \n",
       "\n",
       "[237 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/KS/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/KS-day/2018-jan-day-KS.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-01-29 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2018-01-26 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-01-08 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-01-17 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2018-01-14 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279087</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-01-30 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279088</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-01-29 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279089</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-01-10 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279090</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-01-24 11:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279091</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-01-09 07:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>274501 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-01-29 19:05:00 UTC   \n",
       "1       7f9688a0b01344c1f2804913546cad73007d0cf4  2018-01-26 15:35:00 UTC   \n",
       "2       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-01-08 17:45:00 UTC   \n",
       "3       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-01-17 18:05:00 UTC   \n",
       "4       af58a413d1a41205bebf1c507761a5ab74a84725  2018-01-14 19:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "279087  842c02deec3009c2808be94f10e56264890a12f4  2018-01-30 14:30:00 UTC   \n",
       "279088  842c02deec3009c2808be94f10e56264890a12f4  2018-01-29 16:25:00 UTC   \n",
       "279089  842c02deec3009c2808be94f10e56264890a12f4  2018-01-10 17:20:00 UTC   \n",
       "279090  842c02deec3009c2808be94f10e56264890a12f4  2018-01-24 11:40:00 UTC   \n",
       "279091  842c02deec3009c2808be94f10e56264890a12f4  2018-01-09 07:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               672                      670   \n",
       "1          heat          hold               734                      740   \n",
       "2          heat          hold               675                      650   \n",
       "3          heat          hold               678                      680   \n",
       "4          heat          hold               694                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "279087     heat          hold               671                      680   \n",
       "279088     heat          hold               687                      680   \n",
       "279089     heat          hold               693                      690   \n",
       "279090     heat          hold               675                      680   \n",
       "279091     heat          hold               693                      690   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           670            KS     Paola                  117   \n",
       "1                           740            KS    Oswego                  106   \n",
       "2                           650            KS     Paola                  117   \n",
       "3                           680            KS     Paola                  117   \n",
       "4                           700            KS     Ellis                  100   \n",
       "...                         ...           ...       ...                  ...   \n",
       "279087                      680            KS  Scranton                  120   \n",
       "279088                      680            KS  Scranton                  120   \n",
       "279089                      690            KS  Scranton                  120   \n",
       "279090                      680            KS  Scranton                  120   \n",
       "279091                      690            KS  Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "279087             False         False            False   \n",
       "279088             False         False            False   \n",
       "279089             False         False            False   \n",
       "279090             False         False            False   \n",
       "279091             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "279087                       Gas  \n",
       "279088                       Gas  \n",
       "279089                       Gas  \n",
       "279090                       Gas  \n",
       "279091                       Gas  \n",
       "\n",
       "[274501 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/KS/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/KS-day/2019-jan-day-KS.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2019-01-05 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>640</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2019-01-05 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>639</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2019-01-15 13:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-01-16 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2019-01-31 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>642</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405084</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-01-03 09:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405085</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-01-28 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>687</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405086</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-01-23 10:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>667</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405087</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-01-22 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405088</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-01-14 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>401636 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       62a890bb783d07af9497a160b6a7f4ae41579c67  2019-01-05 13:50:00 UTC   \n",
       "1       62a890bb783d07af9497a160b6a7f4ae41579c67  2019-01-05 13:10:00 UTC   \n",
       "2       62a890bb783d07af9497a160b6a7f4ae41579c67  2019-01-15 13:25:00 UTC   \n",
       "3       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-01-16 17:45:00 UTC   \n",
       "4       62a890bb783d07af9497a160b6a7f4ae41579c67  2019-01-31 19:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "405084  842c02deec3009c2808be94f10e56264890a12f4  2019-01-03 09:30:00 UTC   \n",
       "405085  842c02deec3009c2808be94f10e56264890a12f4  2019-01-28 13:40:00 UTC   \n",
       "405086  842c02deec3009c2808be94f10e56264890a12f4  2019-01-23 10:35:00 UTC   \n",
       "405087  842c02deec3009c2808be94f10e56264890a12f4  2019-01-22 13:20:00 UTC   \n",
       "405088  842c02deec3009c2808be94f10e56264890a12f4  2019-01-14 12:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               640                      650   \n",
       "1          heat          hold               639                      650   \n",
       "2          heat          hold               659                      670   \n",
       "3          heat          auto               746                      760   \n",
       "4          heat          hold               642                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "405084     heat          auto               669                      670   \n",
       "405085     heat          hold               687                      690   \n",
       "405086     heat          hold               667                      670   \n",
       "405087     heat          hold               697                      700   \n",
       "405088     heat          hold               668                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           640            KS   Murdock                  118   \n",
       "1                           640            KS   Murdock                  118   \n",
       "2                           670            KS   Murdock                  118   \n",
       "3                           760            KS    Oswego                  106   \n",
       "4                           640            KS   Murdock                  118   \n",
       "...                         ...           ...       ...                  ...   \n",
       "405084                      670            KS  Scranton                  120   \n",
       "405085                      690            KS  Scranton                  120   \n",
       "405086                      670            KS  Scranton                  120   \n",
       "405087                      700            KS  Scranton                  120   \n",
       "405088                      670            KS  Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "405084             False         False            False   \n",
       "405085             False         False            False   \n",
       "405086             False         False            False   \n",
       "405087             False         False            False   \n",
       "405088             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "405084                       Gas  \n",
       "405085                       Gas  \n",
       "405086                       Gas  \n",
       "405087                       Gas  \n",
       "405088                       Gas  \n",
       "\n",
       "[401636 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/KS/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/KS-day/2020-jan-day-KS.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-01-05 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-01-12 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2020-01-13 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2020-01-11 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-01-11 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439140</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-01-02 09:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>649</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439141</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-01-15 15:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439142</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-01-07 08:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439143</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-01-14 07:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439144</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-01-06 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>431238 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-01-05 18:10:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-01-12 14:15:00 UTC   \n",
       "2       e73d2071803a834c945f4eedd70daf1ac7d08fea  2020-01-13 15:55:00 UTC   \n",
       "3       af58a413d1a41205bebf1c507761a5ab74a84725  2020-01-11 14:05:00 UTC   \n",
       "4       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-01-11 12:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "439140  842c02deec3009c2808be94f10e56264890a12f4  2020-01-02 09:10:00 UTC   \n",
       "439141  842c02deec3009c2808be94f10e56264890a12f4  2020-01-15 15:20:00 UTC   \n",
       "439142  842c02deec3009c2808be94f10e56264890a12f4  2020-01-07 08:40:00 UTC   \n",
       "439143  842c02deec3009c2808be94f10e56264890a12f4  2020-01-14 07:40:00 UTC   \n",
       "439144  842c02deec3009c2808be94f10e56264890a12f4  2020-01-06 13:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               682                      700   \n",
       "1          heat          hold               676                      680   \n",
       "2          heat          auto               713                      720   \n",
       "3          auto          hold               699                      750   \n",
       "4          heat          hold               675                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "439140     heat          hold               649                      650   \n",
       "439141     heat          hold               695                      700   \n",
       "439142     heat          hold               655                      650   \n",
       "439143     heat          auto               697                      650   \n",
       "439144     heat          hold               701                      700   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           700            KS    Tooeka                   67   \n",
       "1                           680            KS    Tooeka                   67   \n",
       "2                           720            KS     Paola                   99   \n",
       "3                           700            KS     Ellis                  100   \n",
       "4                           700            KS    Tooeka                   67   \n",
       "...                         ...           ...       ...                  ...   \n",
       "439140                      640            KS  Scranton                  120   \n",
       "439141                      700            KS  Scranton                  120   \n",
       "439142                      650            KS  Scranton                  120   \n",
       "439143                      650            KS  Scranton                  120   \n",
       "439144                      700            KS  Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "439140             False         False            False   \n",
       "439141             False         False            False   \n",
       "439142             False         False            False   \n",
       "439143             False         False            False   \n",
       "439144             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "439140                       Gas  \n",
       "439141                       Gas  \n",
       "439142                       Gas  \n",
       "439143                       Gas  \n",
       "439144                       Gas  \n",
       "\n",
       "[431238 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/KS/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/KS-day/2021-jan-day-KS.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-01-15 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2021-01-09 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2021-01-24 15:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>630</td>\n",
       "      <td>740</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-01-30 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2021-01-23 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>639</td>\n",
       "      <td>740</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288751</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-01-28 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288752</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-01-11 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288753</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-01-04 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288754</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-01-18 14:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288755</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-01-27 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285239 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-01-15 18:20:00 UTC   \n",
       "1       e73d2071803a834c945f4eedd70daf1ac7d08fea  2021-01-09 17:15:00 UTC   \n",
       "2       62a890bb783d07af9497a160b6a7f4ae41579c67  2021-01-24 15:10:00 UTC   \n",
       "3       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-01-30 17:00:00 UTC   \n",
       "4       62a890bb783d07af9497a160b6a7f4ae41579c67  2021-01-23 13:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "288751  842c02deec3009c2808be94f10e56264890a12f4  2021-01-28 15:05:00 UTC   \n",
       "288752  842c02deec3009c2808be94f10e56264890a12f4  2021-01-11 13:20:00 UTC   \n",
       "288753  842c02deec3009c2808be94f10e56264890a12f4  2021-01-04 19:35:00 UTC   \n",
       "288754  842c02deec3009c2808be94f10e56264890a12f4  2021-01-18 14:40:00 UTC   \n",
       "288755  842c02deec3009c2808be94f10e56264890a12f4  2021-01-27 17:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               702                      700   \n",
       "1          heat          hold               706                      710   \n",
       "2          auto          hold               630                      740   \n",
       "3          heat          hold               696                      700   \n",
       "4          auto          hold               639                      740   \n",
       "...         ...           ...               ...                      ...   \n",
       "288751     heat          hold               685                      690   \n",
       "288752     heat          hold               691                      690   \n",
       "288753     heat          hold               671                      670   \n",
       "288754     heat          hold               684                      690   \n",
       "288755     heat          hold               672                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           700            KS    Tooeka                   67   \n",
       "1                           710            KS     Paola                   99   \n",
       "2                           640            KS   Murdock                  118   \n",
       "3                           700            KS    Tooeka                   67   \n",
       "4                           650            KS   Murdock                  118   \n",
       "...                         ...           ...       ...                  ...   \n",
       "288751                      690            KS  Scranton                  120   \n",
       "288752                      690            KS  Scranton                  120   \n",
       "288753                      670            KS  Scranton                  120   \n",
       "288754                      690            KS  Scranton                  120   \n",
       "288755                      670            KS  Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "288751             False         False            False   \n",
       "288752             False         False            False   \n",
       "288753             False         False            False   \n",
       "288754             False         False            False   \n",
       "288755             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "288751                       Gas  \n",
       "288752                       Gas  \n",
       "288753                       Gas  \n",
       "288754                       Gas  \n",
       "288755                       Gas  \n",
       "\n",
       "[285239 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/KS/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>669.450980</td>\n",
       "      <td>820.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>688.583048</td>\n",
       "      <td>799.958904</td>\n",
       "      <td>690.002568</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>652.261905</td>\n",
       "      <td>713.261905</td>\n",
       "      <td>713.142857</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>697.493721</td>\n",
       "      <td>698.607535</td>\n",
       "      <td>698.607535</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>auto</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>691.707317</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>691.846491</td>\n",
       "      <td>693.903509</td>\n",
       "      <td>693.903509</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>747.720698</td>\n",
       "      <td>750.708229</td>\n",
       "      <td>750.708229</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>703.534555</td>\n",
       "      <td>709.582015</td>\n",
       "      <td>709.473772</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>ff0c14deff346a8a3dc87a89fb1ab6632be73334</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tonganoxie</td>\n",
       "      <td>675.750000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>409</th>\n",
       "      <td>ff926605815f441236df0441160fb16f5dc55311</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>685.750000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>689.250000</td>\n",
       "      <td>28.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2596 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year     HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   Jan  2017         auto   \n",
       "1    01a0d7a563a1755d37bb856a01a56e452deb2010   Jan  2017         auto   \n",
       "2    01a0d7a563a1755d37bb856a01a56e452deb2010   Jan  2017         cool   \n",
       "3    01a0d7a563a1755d37bb856a01a56e452deb2010   Jan  2017         heat   \n",
       "4    0448b121ebca96144ab5e5feb5ab88942e98de42   Jan  2017  auxHeatOnly   \n",
       "..                                        ...   ...   ...          ...   \n",
       "405  fdf82bf64cb1a67c28f91aa7246724223df57851   Jan  2021         heat   \n",
       "406  fe779904e8eede7395997bd2762cd654983d363a   Jan  2021         heat   \n",
       "407  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   Jan  2021         heat   \n",
       "408  ff0c14deff346a8a3dc87a89fb1ab6632be73334   Jan  2021         heat   \n",
       "409  ff926605815f441236df0441160fb16f5dc55311   Jan  2021         auto   \n",
       "\n",
       "    CalendarEvent         City  Temperature_ctrl_ave  \\\n",
       "0            auto      Fairway            669.450980   \n",
       "1            hold       Olathe            688.583048   \n",
       "2            hold       Olathe            652.261905   \n",
       "3            hold       Olathe            697.493721   \n",
       "4            auto       Olathe            691.707317   \n",
       "..            ...          ...                   ...   \n",
       "405          hold     Atchison            691.846491   \n",
       "406          hold      Andover            747.720698   \n",
       "407          hold  Eastborough            703.534555   \n",
       "408          hold   Tonganoxie            675.750000   \n",
       "409          hold      Wichita            685.750000   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     820.000000                   640.000000   \n",
       "1                     799.958904                   690.002568   \n",
       "2                     713.261905                   713.142857   \n",
       "3                     698.607535                   698.607535   \n",
       "4                     780.000000                   690.000000   \n",
       "..                           ...                          ...   \n",
       "405                   693.903509                   693.903509   \n",
       "406                   750.708229                   750.708229   \n",
       "407                   709.582015                   709.473772   \n",
       "408                   680.000000                   680.000000   \n",
       "409                   740.000000                   689.250000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   30.0             False         False            False  \n",
       "2                   30.0             False         False            False  \n",
       "3                   30.0             False         False            False  \n",
       "4                   10.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "405                 40.0             False         False            False  \n",
       "406                  0.0             False         False            False  \n",
       "407                 90.0             False         False            False  \n",
       "408                 10.0             False         False            False  \n",
       "409                 28.0             False         False            False  \n",
       "\n",
       "[2596 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/jan/\" + file)\n",
    "    KS_jan = pd.concat([KS_jan, df])\n",
    "    \n",
    "KS_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_jan.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/KS-day/2017-feb-day-KS.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-02-21 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-02-22 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-02-11 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-02-22 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-02-28 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83063</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-02-19 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>645</td>\n",
       "      <td>720</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83064</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-02-19 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>657</td>\n",
       "      <td>720</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83065</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-02-25 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83066</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-02-12 16:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>647</td>\n",
       "      <td>740</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83067</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-02-19 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>652</td>\n",
       "      <td>720</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80774 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier                date_time  \\\n",
       "0      7f9688a0b01344c1f2804913546cad73007d0cf4  2017-02-21 19:35:00 UTC   \n",
       "1      7f9688a0b01344c1f2804913546cad73007d0cf4  2017-02-22 18:20:00 UTC   \n",
       "2      7f9688a0b01344c1f2804913546cad73007d0cf4  2017-02-11 17:50:00 UTC   \n",
       "3      7f9688a0b01344c1f2804913546cad73007d0cf4  2017-02-22 19:50:00 UTC   \n",
       "4      7f9688a0b01344c1f2804913546cad73007d0cf4  2017-02-28 15:30:00 UTC   \n",
       "...                                         ...                      ...   \n",
       "83063  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-02-19 16:25:00 UTC   \n",
       "83064  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-02-19 18:20:00 UTC   \n",
       "83065  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-02-25 19:10:00 UTC   \n",
       "83066  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-02-12 16:20:00 UTC   \n",
       "83067  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-02-19 17:45:00 UTC   \n",
       "\n",
       "      HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0         heat          auto               752                      820   \n",
       "1         heat          auto               751                      820   \n",
       "2         heat          auto               760                      820   \n",
       "3         heat          auto               757                      820   \n",
       "4         heat          auto               743                      820   \n",
       "...        ...           ...               ...                      ...   \n",
       "83063     auto          auto               645                      720   \n",
       "83064     auto          auto               657                      720   \n",
       "83065     auto          auto               677                      740   \n",
       "83066     auto          auto               647                      740   \n",
       "83067     auto          auto               652                      720   \n",
       "\n",
       "       TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                          650            KS    Oswego                  106   \n",
       "1                          650            KS    Oswego                  106   \n",
       "2                          650            KS    Oswego                  106   \n",
       "3                          650            KS    Oswego                  106   \n",
       "4                          650            KS    Oswego                  106   \n",
       "...                        ...           ...       ...                  ...   \n",
       "83063                      620            KS  Lawrence                  120   \n",
       "83064                      620            KS  Lawrence                  120   \n",
       "83065                      680            KS  Lawrence                  120   \n",
       "83066                      650            KS  Lawrence                  120   \n",
       "83067                      620            KS  Lawrence                  120   \n",
       "\n",
       "       allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                 False         False            False   \n",
       "1                 False         False            False   \n",
       "2                 False         False            False   \n",
       "3                 False         False            False   \n",
       "4                 False         False            False   \n",
       "...                 ...           ...              ...   \n",
       "83063             False         False            False   \n",
       "83064             False         False            False   \n",
       "83065             False         False            False   \n",
       "83066             False         False            False   \n",
       "83067             False         False            False   \n",
       "\n",
       "      Auxilliary_Heat_Fuel_Type  \n",
       "0                           Gas  \n",
       "1                           Gas  \n",
       "2                           Gas  \n",
       "3                           Gas  \n",
       "4                           Gas  \n",
       "...                         ...  \n",
       "83063                       Gas  \n",
       "83064                       Gas  \n",
       "83065                       Gas  \n",
       "83066                       Gas  \n",
       "83067                       Gas  \n",
       "\n",
       "[80774 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/KS/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/KS-day/2018-feb-day-KS.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-02-13 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-02-20 10:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>636</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-02-06 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>629</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-02-26 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>643</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2018-02-23 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>647</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255665</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-02-04 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255666</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-02-04 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255667</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-02-04 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>646</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255668</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-02-04 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>647</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255669</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-02-04 19:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>253303 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-02-13 12:40:00 UTC   \n",
       "1       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-02-20 10:05:00 UTC   \n",
       "2       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-02-06 12:50:00 UTC   \n",
       "3       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-02-26 13:15:00 UTC   \n",
       "4       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2018-02-23 13:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "255665  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-02-04 19:30:00 UTC   \n",
       "255666  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-02-04 19:50:00 UTC   \n",
       "255667  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-02-04 17:10:00 UTC   \n",
       "255668  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-02-04 17:05:00 UTC   \n",
       "255669  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-02-04 19:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               660                      670   \n",
       "1          heat          hold               636                      650   \n",
       "2          heat          auto               629                      650   \n",
       "3          heat          hold               643                      670   \n",
       "4          heat          hold               647                      670   \n",
       "...         ...           ...               ...                      ...   \n",
       "255665     auto          auto               688                      760   \n",
       "255666     auto          auto               690                      760   \n",
       "255667     auto          auto               646                      820   \n",
       "255668     auto          auto               647                      820   \n",
       "255669     auto          auto               684                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           670            KS     Paola                  117   \n",
       "1                           640            KS     Paola                  117   \n",
       "2                           630            KS     Paola                  117   \n",
       "3                           670            KS     Paola                  117   \n",
       "4                           670            KS     Paola                  117   \n",
       "...                         ...           ...       ...                  ...   \n",
       "255665                      690            KS  Lawrence                  120   \n",
       "255666                      690            KS  Lawrence                  120   \n",
       "255667                      620            KS  Lawrence                  120   \n",
       "255668                      620            KS  Lawrence                  120   \n",
       "255669                      690            KS  Lawrence                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "255665             False         False            False   \n",
       "255666             False         False            False   \n",
       "255667             False         False            False   \n",
       "255668             False         False            False   \n",
       "255669             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "255665                       Gas  \n",
       "255666                       Gas  \n",
       "255667                       Gas  \n",
       "255668                       Gas  \n",
       "255669                       Gas  \n",
       "\n",
       "[253303 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/KS/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/KS-day/2019-feb-day-KS.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2019-02-18 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2019-02-18 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-02-21 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2019-02-12 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>684</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2019-02-24 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275215</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-02-28 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275216</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-02-07 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>659</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275217</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-02-13 11:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>671</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275218</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-02-05 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>677</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275219</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-02-07 10:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>271506 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7bf217daebd9144ce543a1754527180e81166ccf  2019-02-18 13:35:00 UTC   \n",
       "1       7bf217daebd9144ce543a1754527180e81166ccf  2019-02-18 13:30:00 UTC   \n",
       "2       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-02-21 16:05:00 UTC   \n",
       "3       7bf217daebd9144ce543a1754527180e81166ccf  2019-02-12 13:35:00 UTC   \n",
       "4       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2019-02-24 16:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "275215  842c02deec3009c2808be94f10e56264890a12f4  2019-02-28 14:20:00 UTC   \n",
       "275216  842c02deec3009c2808be94f10e56264890a12f4  2019-02-07 12:50:00 UTC   \n",
       "275217  842c02deec3009c2808be94f10e56264890a12f4  2019-02-13 11:10:00 UTC   \n",
       "275218  842c02deec3009c2808be94f10e56264890a12f4  2019-02-05 16:05:00 UTC   \n",
       "275219  842c02deec3009c2808be94f10e56264890a12f4  2019-02-07 10:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               677                      750   \n",
       "1          auto          auto               675                      750   \n",
       "2          heat          hold               749                      760   \n",
       "3          auto          hold               684                      750   \n",
       "4          heat          hold               681                      680   \n",
       "...         ...           ...               ...                      ...   \n",
       "275215     heat          hold               701                      720   \n",
       "275216     heat          hold               659                      700   \n",
       "275217     heat          auto               671                      670   \n",
       "275218     heat          auto               677                      690   \n",
       "275219     heat          hold               664                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           680            KS    El Dorado   \n",
       "1                           680            KS    El Dorado   \n",
       "2                           760            KS       Oswego   \n",
       "3                           680            KS    El Dorado   \n",
       "4                           680            KS  Eastborough   \n",
       "...                         ...           ...          ...   \n",
       "275215                      720            KS     Scranton   \n",
       "275216                      700            KS     Scranton   \n",
       "275217                      670            KS     Scranton   \n",
       "275218                      690            KS     Scranton   \n",
       "275219                      670            KS     Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        67             False         False            False   \n",
       "1                        67             False         False            False   \n",
       "2                       106             False         False            False   \n",
       "3                        67             False         False            False   \n",
       "4                        79              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "275215                  120             False         False            False   \n",
       "275216                  120             False         False            False   \n",
       "275217                  120             False         False            False   \n",
       "275218                  120             False         False            False   \n",
       "275219                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "275215                       Gas  \n",
       "275216                       Gas  \n",
       "275217                       Gas  \n",
       "275218                       Gas  \n",
       "275219                       Gas  \n",
       "\n",
       "[271506 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/KS/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/KS-day/2020-feb-day-KS.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2020-02-06 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>780</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2020-02-09 17:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2020-02-17 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-02-22 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2020-02-09 19:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398602</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-02-03 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398603</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-02-04 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398604</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-02-13 14:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398605</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-02-20 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398606</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-02-18 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>391582 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7bf217daebd9144ce543a1754527180e81166ccf  2020-02-06 12:30:00 UTC   \n",
       "1       7f9688a0b01344c1f2804913546cad73007d0cf4  2020-02-09 17:55:00 UTC   \n",
       "2       e73d2071803a834c945f4eedd70daf1ac7d08fea  2020-02-17 17:00:00 UTC   \n",
       "3       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-02-22 18:15:00 UTC   \n",
       "4       7f9688a0b01344c1f2804913546cad73007d0cf4  2020-02-09 19:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "398602  842c02deec3009c2808be94f10e56264890a12f4  2020-02-03 15:40:00 UTC   \n",
       "398603  842c02deec3009c2808be94f10e56264890a12f4  2020-02-04 15:05:00 UTC   \n",
       "398604  842c02deec3009c2808be94f10e56264890a12f4  2020-02-13 14:25:00 UTC   \n",
       "398605  842c02deec3009c2808be94f10e56264890a12f4  2020-02-20 14:10:00 UTC   \n",
       "398606  842c02deec3009c2808be94f10e56264890a12f4  2020-02-18 17:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               707                      780   \n",
       "1          heat          hold               747                      730   \n",
       "2          heat          hold               716                      720   \n",
       "3          heat          hold               678                      680   \n",
       "4          heat          hold               729                      730   \n",
       "...         ...           ...               ...                      ...   \n",
       "398602     heat          hold               685                      690   \n",
       "398603     heat          hold               699                      700   \n",
       "398604     heat          hold               688                      700   \n",
       "398605     heat          hold               697                      710   \n",
       "398606     heat          hold               715                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState       City  Age_of_Home__years_  \\\n",
       "0                           710            KS  El Dorado                   67   \n",
       "1                           730            KS     Oswego                  106   \n",
       "2                           720            KS      Paola                   99   \n",
       "3                           680            KS     Tooeka                   67   \n",
       "4                           730            KS     Oswego                  106   \n",
       "...                         ...           ...        ...                  ...   \n",
       "398602                      690            KS   Scranton                  120   \n",
       "398603                      700            KS   Scranton                  120   \n",
       "398604                      700            KS   Scranton                  120   \n",
       "398605                      710            KS   Scranton                  120   \n",
       "398606                      670            KS   Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "398602             False         False            False   \n",
       "398603             False         False            False   \n",
       "398604             False         False            False   \n",
       "398605             False         False            False   \n",
       "398606             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "398602                       Gas  \n",
       "398603                       Gas  \n",
       "398604                       Gas  \n",
       "398605                       Gas  \n",
       "398606                       Gas  \n",
       "\n",
       "[391582 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/KS/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/KS-day/2021-feb-day-KS.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2021-02-20 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2021-02-07 16:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2021-02-27 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2021-02-17 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-02-10 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267597</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-02-11 14:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267598</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-02-15 10:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>656</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267599</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-02-01 19:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>672</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267600</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-02-03 09:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>646</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267601</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-02-18 07:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>657</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>264422 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e73d2071803a834c945f4eedd70daf1ac7d08fea  2021-02-20 17:15:00 UTC   \n",
       "1       e73d2071803a834c945f4eedd70daf1ac7d08fea  2021-02-07 16:55:00 UTC   \n",
       "3       fc006352113dcc3ae14023c9e52f72483f64b65e  2021-02-27 18:10:00 UTC   \n",
       "5       e73d2071803a834c945f4eedd70daf1ac7d08fea  2021-02-17 15:40:00 UTC   \n",
       "6       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-02-10 11:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "267597  842c02deec3009c2808be94f10e56264890a12f4  2021-02-11 14:05:00 UTC   \n",
       "267598  842c02deec3009c2808be94f10e56264890a12f4  2021-02-15 10:40:00 UTC   \n",
       "267599  842c02deec3009c2808be94f10e56264890a12f4  2021-02-01 19:25:00 UTC   \n",
       "267600  842c02deec3009c2808be94f10e56264890a12f4  2021-02-03 09:55:00 UTC   \n",
       "267601  842c02deec3009c2808be94f10e56264890a12f4  2021-02-18 07:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               727                      730   \n",
       "1          heat          hold               705                      720   \n",
       "3          heat          hold               664                      650   \n",
       "5          heat          hold               691                      710   \n",
       "6          heat          hold               693                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "267597     heat          hold               664                      670   \n",
       "267598     heat          hold               656                      670   \n",
       "267599     heat          hold               672                      660   \n",
       "267600     heat          hold               646                      650   \n",
       "267601     heat          hold               657                      660   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           730            KS          Paola   \n",
       "1                           720            KS          Paola   \n",
       "3                           650            KS  Arkansas City   \n",
       "5                           710            KS          Paola   \n",
       "6                           700            KS         Tooeka   \n",
       "...                         ...           ...            ...   \n",
       "267597                      670            KS       Scranton   \n",
       "267598                      670            KS       Scranton   \n",
       "267599                      660            KS       Scranton   \n",
       "267600                      650            KS       Scranton   \n",
       "267601                      660            KS       Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        99             False         False            False   \n",
       "1                        99             False         False            False   \n",
       "3                       100              True         False            False   \n",
       "5                        99             False         False            False   \n",
       "6                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "267597                  120             False         False            False   \n",
       "267598                  120             False         False            False   \n",
       "267599                  120             False         False            False   \n",
       "267600                  120             False         False            False   \n",
       "267601                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "6                            Gas  \n",
       "...                          ...  \n",
       "267597                       Gas  \n",
       "267598                       Gas  \n",
       "267599                       Gas  \n",
       "267600                       Gas  \n",
       "267601                       Gas  \n",
       "\n",
       "[264422 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/KS/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>706.272727</td>\n",
       "      <td>787.878788</td>\n",
       "      <td>687.878788</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>700.478645</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>692.076023</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>653.779614</td>\n",
       "      <td>650.000000</td>\n",
       "      <td>630.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auxHeatOnly</td>\n",
       "      <td>auto</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>699.107143</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>674.253521</td>\n",
       "      <td>661.408451</td>\n",
       "      <td>661.408451</td>\n",
       "      <td>100.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>fdadb8143c9f8d15c6bfbffab7628b015c5db3cb</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>702.958946</td>\n",
       "      <td>755.431288</td>\n",
       "      <td>704.079516</td>\n",
       "      <td>35.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>426</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>693.121196</td>\n",
       "      <td>695.846739</td>\n",
       "      <td>695.847283</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>753.238827</td>\n",
       "      <td>756.745810</td>\n",
       "      <td>756.745810</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>699.762640</td>\n",
       "      <td>708.634831</td>\n",
       "      <td>708.634831</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2544 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year     HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   feb  2017         auto   \n",
       "1    01a0d7a563a1755d37bb856a01a56e452deb2010   feb  2017         auto   \n",
       "2    01a0d7a563a1755d37bb856a01a56e452deb2010   feb  2017         heat   \n",
       "3    025ec7c58b524de74d8986f4896917c31052701a   feb  2017         heat   \n",
       "4    0448b121ebca96144ab5e5feb5ab88942e98de42   feb  2017  auxHeatOnly   \n",
       "..                                        ...   ...   ...          ...   \n",
       "424  fc006352113dcc3ae14023c9e52f72483f64b65e   feb  2021         heat   \n",
       "425  fdadb8143c9f8d15c6bfbffab7628b015c5db3cb   feb  2021         auto   \n",
       "426  fdf82bf64cb1a67c28f91aa7246724223df57851   feb  2021         heat   \n",
       "427  fe779904e8eede7395997bd2762cd654983d363a   feb  2021         heat   \n",
       "428  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   feb  2021         heat   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold        Fairway            706.272727   \n",
       "1            hold         Olathe            700.478645   \n",
       "2            hold         Olathe            692.076023   \n",
       "3            auto  Overland Park            653.779614   \n",
       "4            auto         Olathe            699.107143   \n",
       "..            ...            ...                   ...   \n",
       "424          hold  Arkansas City            674.253521   \n",
       "425          hold         Olathe            702.958946   \n",
       "426          hold       Atchison            693.121196   \n",
       "427          hold        Andover            753.238827   \n",
       "428          hold    Eastborough            699.762640   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     787.878788                   687.878788   \n",
       "1                     780.000000                   700.000000   \n",
       "2                     690.000000                   690.000000   \n",
       "3                     650.000000                   630.000000   \n",
       "4                     780.000000                   700.000000   \n",
       "..                           ...                          ...   \n",
       "424                   661.408451                   661.408451   \n",
       "425                   755.431288                   704.079516   \n",
       "426                   695.846739                   695.847283   \n",
       "427                   756.745810                   756.745810   \n",
       "428                   708.634831                   708.634831   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   30.0             False         False            False  \n",
       "2                   30.0             False         False            False  \n",
       "3                    5.0             False         False             True  \n",
       "4                   10.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "424                100.0              True         False            False  \n",
       "425                 35.0             False         False            False  \n",
       "426                 40.0             False         False            False  \n",
       "427                  0.0             False         False            False  \n",
       "428                 90.0             False         False            False  \n",
       "\n",
       "[2544 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/feb/\" + file)\n",
    "    KS_feb = pd.concat([KS_feb, df])\n",
    "    \n",
    "KS_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_feb.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/KS-day/2017-jun-day-KS.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-06-16 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-06-08 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-06-11 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-06-18 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-06-28 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>771</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173561</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-06-30 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>775</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173562</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-06-20 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173563</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-06-20 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173564</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-06-21 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173565</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-06-20 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>167431 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-06-16 15:00:00 UTC   \n",
       "1       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-06-08 12:50:00 UTC   \n",
       "2       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-06-11 14:40:00 UTC   \n",
       "3       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-06-18 17:20:00 UTC   \n",
       "4       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-06-28 19:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "173561  842c02deec3009c2808be94f10e56264890a12f4  2017-06-30 19:35:00 UTC   \n",
       "173562  842c02deec3009c2808be94f10e56264890a12f4  2017-06-20 13:20:00 UTC   \n",
       "173563  842c02deec3009c2808be94f10e56264890a12f4  2017-06-20 17:20:00 UTC   \n",
       "173564  842c02deec3009c2808be94f10e56264890a12f4  2017-06-21 16:15:00 UTC   \n",
       "173565  842c02deec3009c2808be94f10e56264890a12f4  2017-06-20 14:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               755                      760   \n",
       "1          cool          hold               741                      780   \n",
       "2          cool          hold               760                      750   \n",
       "3          cool          hold               765                      780   \n",
       "4          cool          hold               771                      750   \n",
       "...         ...           ...               ...                      ...   \n",
       "173561     cool          auto               775                      780   \n",
       "173562     cool          auto               736                      750   \n",
       "173563     cool          hold               770                      780   \n",
       "173564     cool          hold               779                      780   \n",
       "173565     cool          hold               743                      780   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           760            KS        Paola   \n",
       "1                           780            KS  Eastborough   \n",
       "2                           750            KS        Paola   \n",
       "3                           780            KS        Paola   \n",
       "4                           750            KS  Eastborough   \n",
       "...                         ...           ...          ...   \n",
       "173561                      760            KS     Scranton   \n",
       "173562                      760            KS     Scranton   \n",
       "173563                      760            KS     Scranton   \n",
       "173564                      760            KS     Scranton   \n",
       "173565                      760            KS     Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       117             False         False            False   \n",
       "1                        79              True         False            False   \n",
       "2                       117             False         False            False   \n",
       "3                       117             False         False            False   \n",
       "4                        79              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "173561                  120             False         False            False   \n",
       "173562                  120             False         False            False   \n",
       "173563                  120             False         False            False   \n",
       "173564                  120             False         False            False   \n",
       "173565                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "173561                       Gas  \n",
       "173562                       Gas  \n",
       "173563                       Gas  \n",
       "173564                       Gas  \n",
       "173565                       Gas  \n",
       "\n",
       "[167431 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/KS/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/KS-day/2018-jun-day-KS.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>781ec8442b037999e55ca700ac584e460a0304d6</td>\n",
       "      <td>2018-06-14 11:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>730</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Salina</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2018-06-23 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>730</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2018-06-03 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f4ca0722190325549f736870c8f8b5a4560c0708</td>\n",
       "      <td>2018-06-02 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Conway Springs</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f4ca0722190325549f736870c8f8b5a4560c0708</td>\n",
       "      <td>2018-06-08 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Conway Springs</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369468</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-06-25 08:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369469</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-06-11 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369470</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-06-18 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369471</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-06-28 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369472</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-06-25 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>361015 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       781ec8442b037999e55ca700ac584e460a0304d6  2018-06-14 11:40:00 UTC   \n",
       "1       fc006352113dcc3ae14023c9e52f72483f64b65e  2018-06-23 17:15:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2018-06-03 14:30:00 UTC   \n",
       "3       f4ca0722190325549f736870c8f8b5a4560c0708  2018-06-02 18:30:00 UTC   \n",
       "4       f4ca0722190325549f736870c8f8b5a4560c0708  2018-06-08 18:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "369468  842c02deec3009c2808be94f10e56264890a12f4  2018-06-25 08:15:00 UTC   \n",
       "369469  842c02deec3009c2808be94f10e56264890a12f4  2018-06-11 15:05:00 UTC   \n",
       "369470  842c02deec3009c2808be94f10e56264890a12f4  2018-06-18 18:10:00 UTC   \n",
       "369471  842c02deec3009c2808be94f10e56264890a12f4  2018-06-28 13:20:00 UTC   \n",
       "369472  842c02deec3009c2808be94f10e56264890a12f4  2018-06-25 14:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               686                      730   \n",
       "1          auto          hold               728                      730   \n",
       "2          cool          hold               715                      730   \n",
       "3          cool          auto               700                      700   \n",
       "4          cool          auto               698                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "369468     cool          auto               719                      720   \n",
       "369469     cool          hold               759                      770   \n",
       "369470     cool          hold               753                      760   \n",
       "369471     cool          hold               760                      760   \n",
       "369472     cool          hold               734                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           670            KS          Salina   \n",
       "1                           640            KS   Arkansas City   \n",
       "2                           730            KS          Tooeka   \n",
       "3                           700            KS  Conway Springs   \n",
       "4                           700            KS  Conway Springs   \n",
       "...                         ...           ...             ...   \n",
       "369468                      720            KS        Scranton   \n",
       "369469                      770            KS        Scranton   \n",
       "369470                      760            KS        Scranton   \n",
       "369471                      760            KS        Scranton   \n",
       "369472                      770            KS        Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        67             False         False            False   \n",
       "1                       100              True         False            False   \n",
       "2                        67             False         False            False   \n",
       "3                       118              True          True             True   \n",
       "4                       118              True          True             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "369468                  120             False         False            False   \n",
       "369469                  120             False         False            False   \n",
       "369470                  120             False         False            False   \n",
       "369471                  120             False         False            False   \n",
       "369472                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "369468                       Gas  \n",
       "369469                       Gas  \n",
       "369470                       Gas  \n",
       "369471                       Gas  \n",
       "369472                       Gas  \n",
       "\n",
       "[361015 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/KS/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/KS-day/2019-jun-day-KS.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>030a1ce1894ca9d7e049cc41dc57f72e597cf256</td>\n",
       "      <td>2019-06-10 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>770</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2019-06-27 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2019-06-28 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-06-27 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>749</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>030a1ce1894ca9d7e049cc41dc57f72e597cf256</td>\n",
       "      <td>2019-06-07 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>770</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Westwood</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482113</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-06-03 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482114</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-06-12 07:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482115</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-06-20 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482116</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-06-10 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482117</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-06-19 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>459819 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       030a1ce1894ca9d7e049cc41dc57f72e597cf256  2019-06-10 12:05:00 UTC   \n",
       "1       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2019-06-27 18:35:00 UTC   \n",
       "2       e73d2071803a834c945f4eedd70daf1ac7d08fea  2019-06-28 18:40:00 UTC   \n",
       "3       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-06-27 16:45:00 UTC   \n",
       "4       030a1ce1894ca9d7e049cc41dc57f72e597cf256  2019-06-07 12:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "482113  842c02deec3009c2808be94f10e56264890a12f4  2019-06-03 15:10:00 UTC   \n",
       "482114  842c02deec3009c2808be94f10e56264890a12f4  2019-06-12 07:00:00 UTC   \n",
       "482115  842c02deec3009c2808be94f10e56264890a12f4  2019-06-20 13:05:00 UTC   \n",
       "482116  842c02deec3009c2808be94f10e56264890a12f4  2019-06-10 18:55:00 UTC   \n",
       "482117  842c02deec3009c2808be94f10e56264890a12f4  2019-06-19 12:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               729                      770   \n",
       "1          cool          hold               754                      750   \n",
       "2          cool          hold               746                      740   \n",
       "3          cool          auto               749                      760   \n",
       "4          cool          auto               762                      770   \n",
       "...         ...           ...               ...                      ...   \n",
       "482113     cool          hold               733                      770   \n",
       "482114     cool          hold               746                      800   \n",
       "482115     cool          hold               721                      790   \n",
       "482116     heat          hold               721                      720   \n",
       "482117     cool          hold               712                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           630            KS     Westwood   \n",
       "1                           750            KS  Eastborough   \n",
       "2                           740            KS        Paola   \n",
       "3                           760            KS       Oswego   \n",
       "4                           630            KS     Westwood   \n",
       "...                         ...           ...          ...   \n",
       "482113                      770            KS     Scranton   \n",
       "482114                      800            KS     Scranton   \n",
       "482115                      790            KS     Scranton   \n",
       "482116                      720            KS     Scranton   \n",
       "482117                      800            KS     Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        99              True         False            False   \n",
       "1                        79              True         False            False   \n",
       "2                        99             False         False            False   \n",
       "3                       106             False         False            False   \n",
       "4                        99              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "482113                  120             False         False            False   \n",
       "482114                  120             False         False            False   \n",
       "482115                  120             False         False            False   \n",
       "482116                  120             False         False            False   \n",
       "482117                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "482113                       Gas  \n",
       "482114                       Gas  \n",
       "482115                       Gas  \n",
       "482116                       Gas  \n",
       "482117                       Gas  \n",
       "\n",
       "[459819 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/KS/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/KS-day/2020-jun-day-KS.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-06-21 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>690</td>\n",
       "      <td>615</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2020-06-01 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>669</td>\n",
       "      <td>669</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-06-14 11:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>690</td>\n",
       "      <td>615</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-06-18 10:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>690</td>\n",
       "      <td>615</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2020-06-18 12:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>720</td>\n",
       "      <td>640</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492149</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2020-06-30 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492150</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2020-06-14 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>770</td>\n",
       "      <td>780</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492151</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2020-06-13 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>730</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492152</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2020-06-13 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>780</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>492153</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2020-06-13 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>780</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>477719 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-06-21 18:20:00 UTC   \n",
       "1       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2020-06-01 19:40:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-06-14 11:10:00 UTC   \n",
       "3       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-06-18 10:35:00 UTC   \n",
       "4       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2020-06-18 12:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "492149  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2020-06-30 15:10:00 UTC   \n",
       "492150  a1b0dc41acf5beef99934faecb9e1460d7b78173  2020-06-14 19:45:00 UTC   \n",
       "492151  a1b0dc41acf5beef99934faecb9e1460d7b78173  2020-06-13 19:20:00 UTC   \n",
       "492152  a1b0dc41acf5beef99934faecb9e1460d7b78173  2020-06-13 17:45:00 UTC   \n",
       "492153  a1b0dc41acf5beef99934faecb9e1460d7b78173  2020-06-13 18:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               692                      690   \n",
       "1          cool          hold               689                      669   \n",
       "2          auto          hold               693                      690   \n",
       "3          auto          hold               691                      690   \n",
       "4          auto          auto               739                      720   \n",
       "...         ...           ...               ...                      ...   \n",
       "492149     cool          auto               723                      700   \n",
       "492150     auto          auto               770                      780   \n",
       "492151     auto          auto               767                      730   \n",
       "492152     auto          auto               755                      780   \n",
       "492153     auto          auto               761                      780   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           615            KS       Tooeka   \n",
       "1                           669            KS  Eastborough   \n",
       "2                           615            KS       Tooeka   \n",
       "3                           615            KS       Tooeka   \n",
       "4                           640            KS   Dodge City   \n",
       "...                         ...           ...          ...   \n",
       "492149                      700            KS      Wichita   \n",
       "492150                      660            KS     Lawrence   \n",
       "492151                      660            KS     Lawrence   \n",
       "492152                      660            KS     Lawrence   \n",
       "492153                      660            KS     Lawrence   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        67             False         False            False   \n",
       "1                        79              True         False            False   \n",
       "2                        67             False         False            False   \n",
       "3                        67             False         False            False   \n",
       "4                       105             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "492149                  120              True         False             True   \n",
       "492150                  120             False         False            False   \n",
       "492151                  120             False         False            False   \n",
       "492152                  120             False         False            False   \n",
       "492153                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "492149                  Electric  \n",
       "492150                       Gas  \n",
       "492151                       Gas  \n",
       "492152                       Gas  \n",
       "492153                       Gas  \n",
       "\n",
       "[477719 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/KS/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/KS-day/2021-jun-day-KS.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2021-06-12 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-06-19 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2021-06-19 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2021-06-11 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2021-06-04 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311073</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-06-03 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311074</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-06-21 07:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311075</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-06-17 08:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311076</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-06-09 08:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311077</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-06-14 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>301419 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       62a890bb783d07af9497a160b6a7f4ae41579c67  2021-06-12 12:55:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-06-19 12:55:00 UTC   \n",
       "2       62a890bb783d07af9497a160b6a7f4ae41579c67  2021-06-19 17:10:00 UTC   \n",
       "3       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2021-06-11 15:40:00 UTC   \n",
       "4       7bf217daebd9144ce543a1754527180e81166ccf  2021-06-04 11:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "311073  842c02deec3009c2808be94f10e56264890a12f4  2021-06-03 13:10:00 UTC   \n",
       "311074  842c02deec3009c2808be94f10e56264890a12f4  2021-06-21 07:40:00 UTC   \n",
       "311075  842c02deec3009c2808be94f10e56264890a12f4  2021-06-17 08:00:00 UTC   \n",
       "311076  842c02deec3009c2808be94f10e56264890a12f4  2021-06-09 08:50:00 UTC   \n",
       "311077  842c02deec3009c2808be94f10e56264890a12f4  2021-06-14 17:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               728                      720   \n",
       "1          cool          hold               695                      690   \n",
       "2          cool          hold               730                      720   \n",
       "3          cool          hold               697                      695   \n",
       "4          auto          hold               710                      710   \n",
       "...         ...           ...               ...                      ...   \n",
       "311073     cool          hold               694                      740   \n",
       "311074     cool          hold               714                      690   \n",
       "311075     cool          hold               735                      680   \n",
       "311076     cool          hold               709                      680   \n",
       "311077     cool          hold               736                      740   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           720            KS      Murdock   \n",
       "1                           690            KS       Tooeka   \n",
       "2                           720            KS      Murdock   \n",
       "3                           695            KS  Eastborough   \n",
       "4                           660            KS    El Dorado   \n",
       "...                         ...           ...          ...   \n",
       "311073                      740            KS     Scranton   \n",
       "311074                      690            KS     Scranton   \n",
       "311075                      680            KS     Scranton   \n",
       "311076                      680            KS     Scranton   \n",
       "311077                      740            KS     Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       118             False         False            False   \n",
       "1                        67             False         False            False   \n",
       "2                       118             False         False            False   \n",
       "3                        79              True         False            False   \n",
       "4                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "311073                  120             False         False            False   \n",
       "311074                  120             False         False            False   \n",
       "311075                  120             False         False            False   \n",
       "311076                  120             False         False            False   \n",
       "311077                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "311073                       Gas  \n",
       "311074                       Gas  \n",
       "311075                       Gas  \n",
       "311076                       Gas  \n",
       "311077                       Gas  \n",
       "\n",
       "[301419 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/KS/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>760.782609</td>\n",
       "      <td>736.869565</td>\n",
       "      <td>736.173913</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>749.553657</td>\n",
       "      <td>754.137520</td>\n",
       "      <td>677.127186</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>721.419820</td>\n",
       "      <td>720.009610</td>\n",
       "      <td>719.966366</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>740.900826</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>743.645833</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>749.312500</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>732.458333</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>705.988506</td>\n",
       "      <td>730.689655</td>\n",
       "      <td>680.517241</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>752.840278</td>\n",
       "      <td>748.888889</td>\n",
       "      <td>748.888889</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>702.469314</td>\n",
       "      <td>696.205776</td>\n",
       "      <td>696.054152</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>ff0c14deff346a8a3dc87a89fb1ab6632be73334</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tonganoxie</td>\n",
       "      <td>654.805556</td>\n",
       "      <td>655.000000</td>\n",
       "      <td>605.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3024 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   jun  2017     cool   \n",
       "1    01a0d7a563a1755d37bb856a01a56e452deb2010   jun  2017     auto   \n",
       "2    025ec7c58b524de74d8986f4896917c31052701a   jun  2017     cool   \n",
       "3    0448b121ebca96144ab5e5feb5ab88942e98de42   jun  2017     cool   \n",
       "4    0448b121ebca96144ab5e5feb5ab88942e98de42   jun  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "431  fc006352113dcc3ae14023c9e52f72483f64b65e   jun  2021     auto   \n",
       "432  fdf82bf64cb1a67c28f91aa7246724223df57851   jun  2021     auto   \n",
       "433  fe779904e8eede7395997bd2762cd654983d363a   jun  2021     cool   \n",
       "434  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   jun  2021     cool   \n",
       "435  ff0c14deff346a8a3dc87a89fb1ab6632be73334   jun  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold        Fairway            760.782609   \n",
       "1            hold         Olathe            749.553657   \n",
       "2            hold  Overland Park            721.419820   \n",
       "3            auto         Olathe            740.900826   \n",
       "4            hold         Olathe            743.645833   \n",
       "..            ...            ...                   ...   \n",
       "431          hold  Arkansas City            732.458333   \n",
       "432          hold       Atchison            705.988506   \n",
       "433          hold        Andover            752.840278   \n",
       "434          hold    Eastborough            702.469314   \n",
       "435          hold     Tonganoxie            654.805556   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     736.869565                   736.173913   \n",
       "1                     754.137520                   677.127186   \n",
       "2                     720.009610                   719.966366   \n",
       "3                     750.000000                   700.000000   \n",
       "4                     750.000000                   749.312500   \n",
       "..                           ...                          ...   \n",
       "431                   730.000000                   670.000000   \n",
       "432                   730.689655                   680.517241   \n",
       "433                   748.888889                   748.888889   \n",
       "434                   696.205776                   696.054152   \n",
       "435                   655.000000                   605.000000   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   30.0             False         False            False  \n",
       "2                    5.0             False         False             True  \n",
       "3                   10.0             False         False            False  \n",
       "4                   10.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "431                100.0              True         False            False  \n",
       "432                 40.0             False         False            False  \n",
       "433                  0.0             False         False            False  \n",
       "434                 90.0             False         False            False  \n",
       "435                 10.0             False         False            False  \n",
       "\n",
       "[3024 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/jun/\" + file)\n",
    "    KS_jun = pd.concat([KS_jun, df])\n",
    "    \n",
    "KS_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_jun.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/KS-day/2017-jul-day-KS.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-07-29 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-07-20 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-07-26 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-07-05 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-07-01 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203528</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-07-08 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>796</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203529</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-07-10 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>786</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203530</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2017-07-02 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203531</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-07-09 16:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>730</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203532</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-07-09 15:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>730</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>197146 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-07-29 18:55:00 UTC   \n",
       "1       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-07-20 17:00:00 UTC   \n",
       "2       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-07-26 19:45:00 UTC   \n",
       "3       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-07-05 19:05:00 UTC   \n",
       "4       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-07-01 19:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "203528  842c02deec3009c2808be94f10e56264890a12f4  2017-07-08 17:30:00 UTC   \n",
       "203529  842c02deec3009c2808be94f10e56264890a12f4  2017-07-10 16:50:00 UTC   \n",
       "203530  842c02deec3009c2808be94f10e56264890a12f4  2017-07-02 17:50:00 UTC   \n",
       "203531  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-07-09 16:40:00 UTC   \n",
       "203532  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-07-09 15:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      760   \n",
       "1          cool          hold               765                      760   \n",
       "2          cool          hold               758                      755   \n",
       "3          cool          hold               748                      750   \n",
       "4          cool          hold               761                      760   \n",
       "...         ...           ...               ...                      ...   \n",
       "203528     cool          hold               796                      770   \n",
       "203529     cool          auto               786                      770   \n",
       "203530     cool          hold               758                      770   \n",
       "203531     auto          auto               739                      730   \n",
       "203532     auto          auto               735                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           760            KS     Paola                  117   \n",
       "1                           760            KS     Paola                  117   \n",
       "2                           755            KS     Paola                  117   \n",
       "3                           750            KS     Paola                  117   \n",
       "4                           760            KS     Paola                  117   \n",
       "...                         ...           ...       ...                  ...   \n",
       "203528                      760            KS  Scranton                  120   \n",
       "203529                      760            KS  Scranton                  120   \n",
       "203530                      760            KS  Scranton                  120   \n",
       "203531                      660            KS  Lawrence                  120   \n",
       "203532                      660            KS  Lawrence                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "203528             False         False            False   \n",
       "203529             False         False            False   \n",
       "203530             False         False            False   \n",
       "203531             False         False            False   \n",
       "203532             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "203528                       Gas  \n",
       "203529                       Gas  \n",
       "203530                       Gas  \n",
       "203531                       Gas  \n",
       "203532                       Gas  \n",
       "\n",
       "[197146 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/KS/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/KS-day/2018-jul-day-KS.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2018-07-28 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2018-07-29 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>794</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2018-07-24 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2018-07-24 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>722</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2018-07-08 16:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>795</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384284</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-07-01 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>854</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384285</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-07-01 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>855</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384286</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-07-01 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>852</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384287</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-07-01 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>853</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384288</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2018-07-01 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>852</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>372391 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       af58a413d1a41205bebf1c507761a5ab74a84725  2018-07-28 15:20:00 UTC   \n",
       "1       af58a413d1a41205bebf1c507761a5ab74a84725  2018-07-29 15:55:00 UTC   \n",
       "2       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2018-07-24 11:10:00 UTC   \n",
       "3       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2018-07-24 15:10:00 UTC   \n",
       "4       af58a413d1a41205bebf1c507761a5ab74a84725  2018-07-08 16:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "384284  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-07-01 17:20:00 UTC   \n",
       "384285  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-07-01 18:30:00 UTC   \n",
       "384286  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-07-01 15:40:00 UTC   \n",
       "384287  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-07-01 15:55:00 UTC   \n",
       "384288  a1b0dc41acf5beef99934faecb9e1460d7b78173  2018-07-01 15:10:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               770                      800   \n",
       "1          cool          hold               794                      800   \n",
       "2          cool          auto               724                      720   \n",
       "3          cool          auto               722                      720   \n",
       "4          cool          hold               795                      800   \n",
       "...         ...           ...               ...                      ...   \n",
       "384284     heat          auto               854                      820   \n",
       "384285     heat          auto               855                      820   \n",
       "384286     heat          auto               852                      820   \n",
       "384287     heat          auto               853                      820   \n",
       "384288     heat          auto               852                      820   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           790            KS       Ellis   \n",
       "1                           790            KS       Ellis   \n",
       "2                           720            KS  Dodge City   \n",
       "3                           720            KS  Dodge City   \n",
       "4                           790            KS       Ellis   \n",
       "...                         ...           ...         ...   \n",
       "384284                      620            KS    Lawrence   \n",
       "384285                      620            KS    Lawrence   \n",
       "384286                      620            KS    Lawrence   \n",
       "384287                      620            KS    Lawrence   \n",
       "384288                      620            KS    Lawrence   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       100             False         False            False   \n",
       "1                       100             False         False            False   \n",
       "2                       105             False         False            False   \n",
       "3                       105             False         False            False   \n",
       "4                       100             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "384284                  120             False         False            False   \n",
       "384285                  120             False         False            False   \n",
       "384286                  120             False         False            False   \n",
       "384287                  120             False         False            False   \n",
       "384288                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "384284                       Gas  \n",
       "384285                       Gas  \n",
       "384286                       Gas  \n",
       "384287                       Gas  \n",
       "384288                       Gas  \n",
       "\n",
       "[372391 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/KS/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/KS-day/2019-jul-day-KS.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2019-07-10 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>720</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2019-07-14 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2019-07-14 17:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>732</td>\n",
       "      <td>672</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-07-27 16:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2019-07-06 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513999</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-07-28 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>720</td>\n",
       "      <td>717</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514000</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-07-21 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514001</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2019-07-28 19:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514002</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2019-07-28 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514003</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2019-07-28 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>487865 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7bf217daebd9144ce543a1754527180e81166ccf  2019-07-10 12:30:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2019-07-14 13:55:00 UTC   \n",
       "2       fc006352113dcc3ae14023c9e52f72483f64b65e  2019-07-14 17:10:00 UTC   \n",
       "3       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-07-27 16:40:00 UTC   \n",
       "4       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2019-07-06 12:55:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "513999  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-07-28 12:45:00 UTC   \n",
       "514000  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-07-21 18:40:00 UTC   \n",
       "514001  a1b0dc41acf5beef99934faecb9e1460d7b78173  2019-07-28 19:10:00 UTC   \n",
       "514002  a1b0dc41acf5beef99934faecb9e1460d7b78173  2019-07-28 19:20:00 UTC   \n",
       "514003  a1b0dc41acf5beef99934faecb9e1460d7b78173  2019-07-28 18:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          auto               725                      720   \n",
       "1          cool          auto               702                      700   \n",
       "2          auto          hold               733                      732   \n",
       "3          cool          hold               737                      710   \n",
       "4          cool          hold               741                      740   \n",
       "...         ...           ...               ...                      ...   \n",
       "513999     cool          auto               718                      720   \n",
       "514000     cool          auto               736                      730   \n",
       "514001     auto          auto               751                      680   \n",
       "514002     auto          auto               751                      680   \n",
       "514003     auto          auto               750                      680   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           650            KS      El Dorado   \n",
       "1                           650            KS         Tooeka   \n",
       "2                           672            KS  Arkansas City   \n",
       "3                           710            KS         Oswego   \n",
       "4                           740            KS         Tooeka   \n",
       "...                         ...           ...            ...   \n",
       "513999                      717            KS        Wichita   \n",
       "514000                      730            KS        Wichita   \n",
       "514001                      630            KS       Lawrence   \n",
       "514002                      630            KS       Lawrence   \n",
       "514003                      630            KS       Lawrence   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        67             False         False            False   \n",
       "1                        67             False         False            False   \n",
       "2                       100              True         False            False   \n",
       "3                       106             False         False            False   \n",
       "4                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "513999                  120              True         False             True   \n",
       "514000                  120              True         False             True   \n",
       "514001                  120             False         False            False   \n",
       "514002                  120             False         False            False   \n",
       "514003                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "513999                  Electric  \n",
       "514000                  Electric  \n",
       "514001                       Gas  \n",
       "514002                       Gas  \n",
       "514003                       Gas  \n",
       "\n",
       "[487865 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/KS/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/KS-day/2020-jul-day-KS.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2020-07-19 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>af58a413d1a41205bebf1c507761a5ab74a84725</td>\n",
       "      <td>2020-07-19 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Ellis</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-07-14 10:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>700</td>\n",
       "      <td>615</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c8dad0e2cdfb05dd04f8dbb5ec56efa00fb9db03</td>\n",
       "      <td>2020-07-18 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>833</td>\n",
       "      <td>840</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Parsons</td>\n",
       "      <td>49</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-07-12 10:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>615</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480669</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-07-15 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480670</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-07-15 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480671</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-07-28 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480672</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-07-13 10:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480673</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-07-08 09:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>673</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>468009 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2020-07-19 12:10:00 UTC   \n",
       "1       af58a413d1a41205bebf1c507761a5ab74a84725  2020-07-19 16:20:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-07-14 10:40:00 UTC   \n",
       "3       c8dad0e2cdfb05dd04f8dbb5ec56efa00fb9db03  2020-07-18 19:05:00 UTC   \n",
       "4       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-07-12 10:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "480669  842c02deec3009c2808be94f10e56264890a12f4  2020-07-15 18:10:00 UTC   \n",
       "480670  842c02deec3009c2808be94f10e56264890a12f4  2020-07-15 14:40:00 UTC   \n",
       "480671  842c02deec3009c2808be94f10e56264890a12f4  2020-07-28 17:50:00 UTC   \n",
       "480672  842c02deec3009c2808be94f10e56264890a12f4  2020-07-13 10:10:00 UTC   \n",
       "480673  842c02deec3009c2808be94f10e56264890a12f4  2020-07-08 09:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               737                      720   \n",
       "1          cool          hold               725                      730   \n",
       "2          auto          hold               704                      700   \n",
       "3          cool          auto               833                      840   \n",
       "4          auto          hold               698                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "480669     cool          auto               736                      740   \n",
       "480670     cool          auto               724                      740   \n",
       "480671     cool          auto               735                      740   \n",
       "480672     cool          hold               683                      680   \n",
       "480673     cool          auto               673                      660   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           720            KS  Dodge City   \n",
       "1                           730            KS       Ellis   \n",
       "2                           615            KS      Tooeka   \n",
       "3                           620            KS     Parsons   \n",
       "4                           615            KS      Tooeka   \n",
       "...                         ...           ...         ...   \n",
       "480669                      660            KS    Scranton   \n",
       "480670                      660            KS    Scranton   \n",
       "480671                      680            KS    Scranton   \n",
       "480672                      680            KS    Scranton   \n",
       "480673                      660            KS    Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       105             False         False            False   \n",
       "1                       100             False         False            False   \n",
       "2                        67             False         False            False   \n",
       "3                        49             False         False            False   \n",
       "4                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "480669                  120             False         False            False   \n",
       "480670                  120             False         False            False   \n",
       "480671                  120             False         False            False   \n",
       "480672                  120             False         False            False   \n",
       "480673                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "480669                       Gas  \n",
       "480670                       Gas  \n",
       "480671                       Gas  \n",
       "480672                       Gas  \n",
       "480673                       Gas  \n",
       "\n",
       "[468009 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/KS/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/KS-day/2021-jul-day-KS.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-07-25 19:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-07-03 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-07-10 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2021-07-18 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2021-07-29 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295917</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-07-15 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295918</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-07-07 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295919</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-07-15 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295920</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-07-07 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295921</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2021-07-07 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>283385 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-07-25 19:50:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-07-03 18:50:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-07-10 18:45:00 UTC   \n",
       "3       62a890bb783d07af9497a160b6a7f4ae41579c67  2021-07-18 12:05:00 UTC   \n",
       "4       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2021-07-29 11:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "295917  842c02deec3009c2808be94f10e56264890a12f4  2021-07-15 13:25:00 UTC   \n",
       "295918  842c02deec3009c2808be94f10e56264890a12f4  2021-07-07 15:45:00 UTC   \n",
       "295919  842c02deec3009c2808be94f10e56264890a12f4  2021-07-15 12:40:00 UTC   \n",
       "295920  842c02deec3009c2808be94f10e56264890a12f4  2021-07-07 15:10:00 UTC   \n",
       "295921  842c02deec3009c2808be94f10e56264890a12f4  2021-07-07 13:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               688                      690   \n",
       "1          cool          hold               690                      690   \n",
       "2          cool          hold               692                      690   \n",
       "3          cool          hold               720                      710   \n",
       "4          cool          hold               689                      690   \n",
       "...         ...           ...               ...                      ...   \n",
       "295917     cool          hold               702                      740   \n",
       "295918     cool          hold               727                      760   \n",
       "295919     cool          hold               692                      740   \n",
       "295920     cool          hold               720                      760   \n",
       "295921     cool          hold               703                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState      City  Age_of_Home__years_  \\\n",
       "0                           690            KS    Tooeka                   67   \n",
       "1                           690            KS    Tooeka                   67   \n",
       "2                           690            KS    Tooeka                   67   \n",
       "3                           710            KS   Murdock                  118   \n",
       "4                           690            KS    Tooeka                   67   \n",
       "...                         ...           ...       ...                  ...   \n",
       "295917                      740            KS  Scranton                  120   \n",
       "295918                      760            KS  Scranton                  120   \n",
       "295919                      740            KS  Scranton                  120   \n",
       "295920                      760            KS  Scranton                  120   \n",
       "295921                      760            KS  Scranton                  120   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "295917             False         False            False   \n",
       "295918             False         False            False   \n",
       "295919             False         False            False   \n",
       "295920             False         False            False   \n",
       "295921             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "295917                       Gas  \n",
       "295918                       Gas  \n",
       "295919                       Gas  \n",
       "295920                       Gas  \n",
       "295921                       Gas  \n",
       "\n",
       "[283385 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/KS/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>769.333333</td>\n",
       "      <td>741.083333</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>749.854839</td>\n",
       "      <td>738.274194</td>\n",
       "      <td>736.677419</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>755.055806</td>\n",
       "      <td>754.794830</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>725.541578</td>\n",
       "      <td>724.058991</td>\n",
       "      <td>719.939588</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>050b44b61d052cae8d52e817cc0f2bb78bb91535</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>731.580645</td>\n",
       "      <td>726.903226</td>\n",
       "      <td>679.677419</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>730.883333</td>\n",
       "      <td>730.100000</td>\n",
       "      <td>669.933333</td>\n",
       "      <td>100.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>694.779134</td>\n",
       "      <td>690.724832</td>\n",
       "      <td>620.043319</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>745.988889</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>384</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>701.937838</td>\n",
       "      <td>698.797297</td>\n",
       "      <td>698.797297</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>ff0c14deff346a8a3dc87a89fb1ab6632be73334</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tonganoxie</td>\n",
       "      <td>653.776042</td>\n",
       "      <td>654.062500</td>\n",
       "      <td>602.812500</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2806 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   jul  2017     cool   \n",
       "1    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   jul  2017     cool   \n",
       "2    01a0d7a563a1755d37bb856a01a56e452deb2010   jul  2017     auto   \n",
       "3    025ec7c58b524de74d8986f4896917c31052701a   jul  2017     cool   \n",
       "4    050b44b61d052cae8d52e817cc0f2bb78bb91535   jul  2017     cool   \n",
       "..                                        ...   ...   ...      ...   \n",
       "381  fc006352113dcc3ae14023c9e52f72483f64b65e   jul  2021     auto   \n",
       "382  fdf82bf64cb1a67c28f91aa7246724223df57851   jul  2021     auto   \n",
       "383  fe779904e8eede7395997bd2762cd654983d363a   jul  2021     cool   \n",
       "384  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   jul  2021     cool   \n",
       "385  ff0c14deff346a8a3dc87a89fb1ab6632be73334   jul  2021     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto        Fairway            769.333333   \n",
       "1            hold        Fairway            749.854839   \n",
       "2            hold         Olathe            755.055806   \n",
       "3            hold  Overland Park            725.541578   \n",
       "4            auto  Overland Park            731.580645   \n",
       "..            ...            ...                   ...   \n",
       "381          hold  Arkansas City            730.883333   \n",
       "382          hold       Atchison            694.779134   \n",
       "383          hold        Andover            745.988889   \n",
       "384          hold    Eastborough            701.937838   \n",
       "385          hold     Tonganoxie            653.776042   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     741.083333                   690.000000   \n",
       "1                     738.274194                   736.677419   \n",
       "2                     754.794830                   690.000000   \n",
       "3                     724.058991                   719.939588   \n",
       "4                     726.903226                   679.677419   \n",
       "..                           ...                          ...   \n",
       "381                   730.100000                   669.933333   \n",
       "382                   690.724832                   620.043319   \n",
       "383                   738.000000                   738.000000   \n",
       "384                   698.797297                   698.797297   \n",
       "385                   654.062500                   602.812500   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   80.0             False         False            False  \n",
       "2                   30.0             False         False            False  \n",
       "3                    5.0             False         False             True  \n",
       "4                   55.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "381                100.0              True         False            False  \n",
       "382                 40.0             False         False            False  \n",
       "383                  0.0             False         False            False  \n",
       "384                 90.0             False         False            False  \n",
       "385                 10.0             False         False            False  \n",
       "\n",
       "[2806 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/jul/\" + file)\n",
    "    KS_jul = pd.concat([KS_jul, df])\n",
    "    \n",
    "KS_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_jul.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/KS-day/2017-aug-day-KS.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-08-23 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-08-29 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-08-22 09:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-08-18 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-08-14 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201029</th>\n",
       "      <td>b17eb28878c2d7b571d48d9558d8145afaa4757a</td>\n",
       "      <td>2017-08-27 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201030</th>\n",
       "      <td>b17eb28878c2d7b571d48d9558d8145afaa4757a</td>\n",
       "      <td>2017-08-10 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201031</th>\n",
       "      <td>b17eb28878c2d7b571d48d9558d8145afaa4757a</td>\n",
       "      <td>2017-08-29 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201032</th>\n",
       "      <td>b17eb28878c2d7b571d48d9558d8145afaa4757a</td>\n",
       "      <td>2017-08-12 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>737</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201033</th>\n",
       "      <td>b17eb28878c2d7b571d48d9558d8145afaa4757a</td>\n",
       "      <td>2017-08-11 14:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>750</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>193698 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-08-23 14:05:00 UTC   \n",
       "1       7f9688a0b01344c1f2804913546cad73007d0cf4  2017-08-29 19:30:00 UTC   \n",
       "2       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-08-22 09:40:00 UTC   \n",
       "3       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-08-18 18:30:00 UTC   \n",
       "4       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-08-14 13:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "201029  b17eb28878c2d7b571d48d9558d8145afaa4757a  2017-08-27 16:00:00 UTC   \n",
       "201030  b17eb28878c2d7b571d48d9558d8145afaa4757a  2017-08-10 19:00:00 UTC   \n",
       "201031  b17eb28878c2d7b571d48d9558d8145afaa4757a  2017-08-29 15:20:00 UTC   \n",
       "201032  b17eb28878c2d7b571d48d9558d8145afaa4757a  2017-08-12 13:50:00 UTC   \n",
       "201033  b17eb28878c2d7b571d48d9558d8145afaa4757a  2017-08-11 14:55:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               736                      750   \n",
       "1          cool          auto               745                      750   \n",
       "2          cool          hold               754                      750   \n",
       "3          cool          hold               756                      770   \n",
       "4          cool          hold               752                      750   \n",
       "...         ...           ...               ...                      ...   \n",
       "201029     auto          hold               749                      750   \n",
       "201030     auto          auto               750                      750   \n",
       "201031     auto          hold               737                      750   \n",
       "201032     auto          auto               737                      750   \n",
       "201033     auto          auto               752                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState    City  Age_of_Home__years_  \\\n",
       "0                           750            KS   Paola                  117   \n",
       "1                           680            KS  Oswego                  106   \n",
       "2                           750            KS   Paola                  117   \n",
       "3                           770            KS   Paola                  117   \n",
       "4                           750            KS   Paola                  117   \n",
       "...                         ...           ...     ...                  ...   \n",
       "201029                      690            KS  Olathe                   20   \n",
       "201030                      690            KS  Olathe                   20   \n",
       "201031                      690            KS  Olathe                   20   \n",
       "201032                      690            KS  Olathe                   20   \n",
       "201033                      690            KS  Olathe                   20   \n",
       "\n",
       "        allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                  False         False            False   \n",
       "1                  False         False            False   \n",
       "2                  False         False            False   \n",
       "3                  False         False            False   \n",
       "4                  False         False            False   \n",
       "...                  ...           ...              ...   \n",
       "201029             False         False            False   \n",
       "201030             False         False            False   \n",
       "201031             False         False            False   \n",
       "201032             False         False            False   \n",
       "201033             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "201029                       Gas  \n",
       "201030                       Gas  \n",
       "201031                       Gas  \n",
       "201032                       Gas  \n",
       "201033                       Gas  \n",
       "\n",
       "[193698 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/KS/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/KS-day/2018-aug-day-KS.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f4ca0722190325549f736870c8f8b5a4560c0708</td>\n",
       "      <td>2018-08-18 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Conway Springs</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2018-08-19 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2018-08-28 16:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2018-08-13 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2018-08-20 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371706</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-08-07 10:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371707</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-08-22 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371708</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-08-07 08:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371709</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-08-20 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371710</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-08-29 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>357580 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f4ca0722190325549f736870c8f8b5a4560c0708  2018-08-18 17:30:00 UTC   \n",
       "1       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2018-08-19 18:55:00 UTC   \n",
       "2       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2018-08-28 16:25:00 UTC   \n",
       "3       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2018-08-13 11:00:00 UTC   \n",
       "4       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2018-08-20 11:35:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "371706  842c02deec3009c2808be94f10e56264890a12f4  2018-08-07 10:15:00 UTC   \n",
       "371707  842c02deec3009c2808be94f10e56264890a12f4  2018-08-22 19:05:00 UTC   \n",
       "371708  842c02deec3009c2808be94f10e56264890a12f4  2018-08-07 08:15:00 UTC   \n",
       "371709  842c02deec3009c2808be94f10e56264890a12f4  2018-08-20 16:50:00 UTC   \n",
       "371710  842c02deec3009c2808be94f10e56264890a12f4  2018-08-29 18:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               702                      700   \n",
       "1          cool          hold               763                      760   \n",
       "2          cool          hold               735                      730   \n",
       "3          cool          hold               737                      740   \n",
       "4          cool          hold               742                      750   \n",
       "...         ...           ...               ...                      ...   \n",
       "371706     cool          hold               701                      700   \n",
       "371707     cool          hold               733                      770   \n",
       "371708     cool          hold               683                      700   \n",
       "371709     cool          auto               735                      780   \n",
       "371710     cool          hold               728                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           690            KS  Conway Springs   \n",
       "1                           760            KS     Eastborough   \n",
       "2                           730            KS      Dodge City   \n",
       "3                           740            KS      Dodge City   \n",
       "4                           750            KS      Dodge City   \n",
       "...                         ...           ...             ...   \n",
       "371706                      700            KS        Scranton   \n",
       "371707                      770            KS        Scranton   \n",
       "371708                      700            KS        Scranton   \n",
       "371709                      780            KS        Scranton   \n",
       "371710                      750            KS        Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       118              True          True             True   \n",
       "1                        79              True         False            False   \n",
       "2                       105             False         False            False   \n",
       "3                       105             False         False            False   \n",
       "4                       105             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "371706                  120             False         False            False   \n",
       "371707                  120             False         False            False   \n",
       "371708                  120             False         False            False   \n",
       "371709                  120             False         False            False   \n",
       "371710                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "371706                       Gas  \n",
       "371707                       Gas  \n",
       "371708                       Gas  \n",
       "371709                       Gas  \n",
       "371710                       Gas  \n",
       "\n",
       "[357580 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/KS/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/KS-day/2019-aug-day-KS.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2019-08-24 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>733</td>\n",
       "      <td>644</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-08-09 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2019-08-31 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2019-08-12 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2019-08-12 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490024</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-08-24 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>726</td>\n",
       "      <td>720</td>\n",
       "      <td>693</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490025</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-08-25 11:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>720</td>\n",
       "      <td>693</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490026</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-08-25 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490027</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-08-25 09:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>693</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490028</th>\n",
       "      <td>7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c</td>\n",
       "      <td>2019-08-10 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>120</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>469017 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fc006352113dcc3ae14023c9e52f72483f64b65e  2019-08-24 15:25:00 UTC   \n",
       "1       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-08-09 15:20:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2019-08-31 15:15:00 UTC   \n",
       "3       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2019-08-12 16:50:00 UTC   \n",
       "4       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2019-08-12 16:10:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "490024  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-08-24 18:25:00 UTC   \n",
       "490025  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-08-25 11:05:00 UTC   \n",
       "490026  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-08-25 17:20:00 UTC   \n",
       "490027  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-08-25 09:30:00 UTC   \n",
       "490028  7aa978dfdfd62ddb38b4a3cbb14f2b26ae1a026c  2019-08-10 18:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               732                      733   \n",
       "1          cool          auto               731                      730   \n",
       "2          cool          hold               686                      690   \n",
       "3          cool          hold               750                      750   \n",
       "4          cool          hold               756                      750   \n",
       "...         ...           ...               ...                      ...   \n",
       "490024     cool          auto               726                      720   \n",
       "490025     cool          auto               729                      720   \n",
       "490026     cool          auto               717                      710   \n",
       "490027     cool          auto               720                      720   \n",
       "490028     cool          auto               710                      710   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           644            KS  Arkansas City   \n",
       "1                           730            KS         Oswego   \n",
       "2                           690            KS         Tooeka   \n",
       "3                           750            KS    Eastborough   \n",
       "4                           750            KS    Eastborough   \n",
       "...                         ...           ...            ...   \n",
       "490024                      693            KS        Wichita   \n",
       "490025                      693            KS        Wichita   \n",
       "490026                      710            KS        Wichita   \n",
       "490027                      693            KS        Wichita   \n",
       "490028                      710            KS        Wichita   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       100              True         False            False   \n",
       "1                       106             False         False            False   \n",
       "2                        67             False         False            False   \n",
       "3                        79              True         False            False   \n",
       "4                        79              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "490024                  120              True         False             True   \n",
       "490025                  120              True         False             True   \n",
       "490026                  120              True         False             True   \n",
       "490027                  120              True         False             True   \n",
       "490028                  120              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "490024                  Electric  \n",
       "490025                  Electric  \n",
       "490026                  Electric  \n",
       "490027                  Electric  \n",
       "490028                  Electric  \n",
       "\n",
       "[469017 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/KS/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/KS-day/2020-aug-day-KS.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2020-08-15 19:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>732</td>\n",
       "      <td>672</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-08-28 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-08-07 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>690</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2020-08-12 13:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2020-08-15 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463907</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-08-31 09:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>680</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463908</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-08-25 07:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>666</td>\n",
       "      <td>666</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463909</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-08-24 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>735</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463910</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-08-31 12:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>463911</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-08-31 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>454801 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fc006352113dcc3ae14023c9e52f72483f64b65e  2020-08-15 19:55:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-08-28 17:25:00 UTC   \n",
       "2       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-08-07 14:35:00 UTC   \n",
       "3       7bf217daebd9144ce543a1754527180e81166ccf  2020-08-12 13:00:00 UTC   \n",
       "4       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2020-08-15 14:50:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "463907  842c02deec3009c2808be94f10e56264890a12f4  2020-08-31 09:25:00 UTC   \n",
       "463908  842c02deec3009c2808be94f10e56264890a12f4  2020-08-25 07:15:00 UTC   \n",
       "463909  842c02deec3009c2808be94f10e56264890a12f4  2020-08-24 15:45:00 UTC   \n",
       "463910  842c02deec3009c2808be94f10e56264890a12f4  2020-08-31 12:35:00 UTC   \n",
       "463911  842c02deec3009c2808be94f10e56264890a12f4  2020-08-31 19:00:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               738                      732   \n",
       "1          cool          hold               696                      695   \n",
       "2          cool          auto               696                      690   \n",
       "3          auto          auto               730                      730   \n",
       "4          cool          hold               703                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "463907     cool          auto               681                      680   \n",
       "463908     cool          hold               681                      666   \n",
       "463909     cool          auto               735                      740   \n",
       "463910     cool          auto               702                      740   \n",
       "463911     cool          hold               727                      730   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           672            KS  Arkansas City   \n",
       "1                           695            KS         Tooeka   \n",
       "2                           650            KS         Tooeka   \n",
       "3                           650            KS      El Dorado   \n",
       "4                           700            KS         Tooeka   \n",
       "...                         ...           ...            ...   \n",
       "463907                      660            KS       Scranton   \n",
       "463908                      666            KS       Scranton   \n",
       "463909                      660            KS       Scranton   \n",
       "463910                      660            KS       Scranton   \n",
       "463911                      730            KS       Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       100              True         False            False   \n",
       "1                        67             False         False            False   \n",
       "2                        67             False         False            False   \n",
       "3                        67             False         False            False   \n",
       "4                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "463907                  120             False         False            False   \n",
       "463908                  120             False         False            False   \n",
       "463909                  120             False         False            False   \n",
       "463910                  120             False         False            False   \n",
       "463911                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "463907                       Gas  \n",
       "463908                       Gas  \n",
       "463909                       Gas  \n",
       "463910                       Gas  \n",
       "463911                       Gas  \n",
       "\n",
       "[454801 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/KS/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>759.714286</td>\n",
       "      <td>746.342857</td>\n",
       "      <td>745.914286</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>753.352660</td>\n",
       "      <td>761.052565</td>\n",
       "      <td>671.215992</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>730.612368</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>733.516129</td>\n",
       "      <td>742.516129</td>\n",
       "      <td>741.741935</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>050b44b61d052cae8d52e817cc0f2bb78bb91535</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>728.400000</td>\n",
       "      <td>749.200000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>684</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>700.440442</td>\n",
       "      <td>699.397290</td>\n",
       "      <td>621.079529</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>705.173611</td>\n",
       "      <td>699.565972</td>\n",
       "      <td>627.795139</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>762.723881</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>687</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>702.521368</td>\n",
       "      <td>692.076923</td>\n",
       "      <td>692.076923</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>710.690476</td>\n",
       "      <td>709.642857</td>\n",
       "      <td>709.642857</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2285 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   aug  2017     cool   \n",
       "1    01a0d7a563a1755d37bb856a01a56e452deb2010   aug  2017     auto   \n",
       "2    025ec7c58b524de74d8986f4896917c31052701a   aug  2017     cool   \n",
       "3    0448b121ebca96144ab5e5feb5ab88942e98de42   aug  2017     cool   \n",
       "4    050b44b61d052cae8d52e817cc0f2bb78bb91535   aug  2017     auto   \n",
       "..                                        ...   ...   ...      ...   \n",
       "684  fdf82bf64cb1a67c28f91aa7246724223df57851   aug  2020     auto   \n",
       "685  fdf82bf64cb1a67c28f91aa7246724223df57851   aug  2020     auto   \n",
       "686  fe779904e8eede7395997bd2762cd654983d363a   aug  2020     cool   \n",
       "687  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   aug  2020     cool   \n",
       "688  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   aug  2020     cool   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            hold        Fairway            759.714286   \n",
       "1            hold         Olathe            753.352660   \n",
       "2            hold  Overland Park            730.612368   \n",
       "3            hold         Olathe            733.516129   \n",
       "4            auto  Overland Park            728.400000   \n",
       "..            ...            ...                   ...   \n",
       "684          auto       Atchison            700.440442   \n",
       "685          hold       Atchison            705.173611   \n",
       "686          hold        Andover            762.723881   \n",
       "687          auto    Eastborough            702.521368   \n",
       "688          hold    Eastborough            710.690476   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     746.342857                   745.914286   \n",
       "1                     761.052565                   671.215992   \n",
       "2                     740.000000                   720.000000   \n",
       "3                     742.516129                   741.741935   \n",
       "4                     749.200000                   680.000000   \n",
       "..                           ...                          ...   \n",
       "684                   699.397290                   621.079529   \n",
       "685                   699.565972                   627.795139   \n",
       "686                   780.000000                   780.000000   \n",
       "687                   692.076923                   692.076923   \n",
       "688                   709.642857                   709.642857   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   30.0             False         False            False  \n",
       "2                    5.0             False         False             True  \n",
       "3                   10.0             False         False            False  \n",
       "4                   55.0             False         False            False  \n",
       "..                   ...               ...           ...              ...  \n",
       "684                 40.0             False         False            False  \n",
       "685                 40.0             False         False            False  \n",
       "686                  0.0             False         False            False  \n",
       "687                 90.0             False         False            False  \n",
       "688                 90.0             False         False            False  \n",
       "\n",
       "[2285 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/aug/\" + file)\n",
    "    KS_aug = pd.concat([KS_aug, df])\n",
    "    \n",
    "KS_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_aug.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/KS-day/2017-dec-day-KS.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>beb1c75205a06d7650da1b7f6c7784ebd2cd0322</td>\n",
       "      <td>2017-12-25 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>79</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2017-12-31 17:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>617</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-12-14 18:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2017-12-13 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6348b0d66bdaf7ddf0f2dbec20785a694ac00335</td>\n",
       "      <td>2017-12-26 11:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>628</td>\n",
       "      <td>650</td>\n",
       "      <td>635</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>117</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260796</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-12-31 15:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>612</td>\n",
       "      <td>730</td>\n",
       "      <td>619</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260797</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-12-31 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>609</td>\n",
       "      <td>730</td>\n",
       "      <td>619</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260798</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-12-17 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260799</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-12-17 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>260800</th>\n",
       "      <td>a1b0dc41acf5beef99934faecb9e1460d7b78173</td>\n",
       "      <td>2017-12-30 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>256389 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       beb1c75205a06d7650da1b7f6c7784ebd2cd0322  2017-12-25 14:15:00 UTC   \n",
       "1       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2017-12-31 17:35:00 UTC   \n",
       "2       7f9688a0b01344c1f2804913546cad73007d0cf4  2017-12-14 18:35:00 UTC   \n",
       "3       7f9688a0b01344c1f2804913546cad73007d0cf4  2017-12-13 18:10:00 UTC   \n",
       "5       6348b0d66bdaf7ddf0f2dbec20785a694ac00335  2017-12-26 11:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "260796  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-12-31 15:20:00 UTC   \n",
       "260797  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-12-31 15:15:00 UTC   \n",
       "260798  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-12-17 19:15:00 UTC   \n",
       "260799  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-12-17 18:55:00 UTC   \n",
       "260800  a1b0dc41acf5beef99934faecb9e1460d7b78173  2017-12-30 19:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               695                      700   \n",
       "1          heat          hold               617                      680   \n",
       "2          heat          hold               756                      760   \n",
       "3          heat          hold               779                      780   \n",
       "5          heat          hold               628                      650   \n",
       "...         ...           ...               ...                      ...   \n",
       "260796     auto          auto               612                      730   \n",
       "260797     auto          auto               609                      730   \n",
       "260798     auto          auto               683                      820   \n",
       "260799     auto          auto               691                      820   \n",
       "260800     auto          auto               675                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           700            KS  Eastborough   \n",
       "1                           680            KS       Tooeka   \n",
       "2                           760            KS       Oswego   \n",
       "3                           780            KS       Oswego   \n",
       "5                           635            KS        Paola   \n",
       "...                         ...           ...          ...   \n",
       "260796                      619            KS     Lawrence   \n",
       "260797                      619            KS     Lawrence   \n",
       "260798                      620            KS     Lawrence   \n",
       "260799                      620            KS     Lawrence   \n",
       "260800                      690            KS     Lawrence   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        79              True         False            False   \n",
       "1                        67             False         False            False   \n",
       "2                       106             False         False            False   \n",
       "3                       106             False         False            False   \n",
       "5                       117             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "260796                  120             False         False            False   \n",
       "260797                  120             False         False            False   \n",
       "260798                  120             False         False            False   \n",
       "260799                  120             False         False            False   \n",
       "260800                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "260796                       Gas  \n",
       "260797                       Gas  \n",
       "260798                       Gas  \n",
       "260799                       Gas  \n",
       "260800                       Gas  \n",
       "\n",
       "[256389 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/KS/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/KS-day/2018-dec-day-KS.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2018-12-26 17:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>628</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2018-12-13 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2018-12-12 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f4ca0722190325549f736870c8f8b5a4560c0708</td>\n",
       "      <td>2018-12-23 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>730</td>\n",
       "      <td>700</td>\n",
       "      <td>KS</td>\n",
       "      <td>Conway Springs</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2018-12-09 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400642</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-12-13 15:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400643</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-12-24 11:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400644</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-12-24 10:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>676</td>\n",
       "      <td>690</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400645</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-12-03 11:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>660</td>\n",
       "      <td>669</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400646</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2018-12-19 09:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>395404 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       62a890bb783d07af9497a160b6a7f4ae41579c67  2018-12-26 17:10:00 UTC   \n",
       "1       62a890bb783d07af9497a160b6a7f4ae41579c67  2018-12-13 18:30:00 UTC   \n",
       "2       62a890bb783d07af9497a160b6a7f4ae41579c67  2018-12-12 16:00:00 UTC   \n",
       "3       f4ca0722190325549f736870c8f8b5a4560c0708  2018-12-23 16:05:00 UTC   \n",
       "4       62a890bb783d07af9497a160b6a7f4ae41579c67  2018-12-09 15:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "400642  842c02deec3009c2808be94f10e56264890a12f4  2018-12-13 15:50:00 UTC   \n",
       "400643  842c02deec3009c2808be94f10e56264890a12f4  2018-12-24 11:20:00 UTC   \n",
       "400644  842c02deec3009c2808be94f10e56264890a12f4  2018-12-24 10:00:00 UTC   \n",
       "400645  842c02deec3009c2808be94f10e56264890a12f4  2018-12-03 11:55:00 UTC   \n",
       "400646  842c02deec3009c2808be94f10e56264890a12f4  2018-12-19 09:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               628                      650   \n",
       "1          heat          hold               676                      680   \n",
       "2          heat          hold               677                      680   \n",
       "3          heat          auto               693                      730   \n",
       "4          heat          hold               652                      670   \n",
       "...         ...           ...               ...                      ...   \n",
       "400642     heat          hold               691                      680   \n",
       "400643     heat          hold               668                      660   \n",
       "400644     heat          auto               676                      690   \n",
       "400645     heat          hold               658                      660   \n",
       "400646     heat          hold               669                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           630            KS         Murdock   \n",
       "1                           680            KS         Murdock   \n",
       "2                           680            KS         Murdock   \n",
       "3                           700            KS  Conway Springs   \n",
       "4                           670            KS         Murdock   \n",
       "...                         ...           ...             ...   \n",
       "400642                      680            KS        Scranton   \n",
       "400643                      660            KS        Scranton   \n",
       "400644                      680            KS        Scranton   \n",
       "400645                      669            KS        Scranton   \n",
       "400646                      670            KS        Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       118             False         False            False   \n",
       "1                       118             False         False            False   \n",
       "2                       118             False         False            False   \n",
       "3                       118              True          True             True   \n",
       "4                       118             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "400642                  120             False         False            False   \n",
       "400643                  120             False         False            False   \n",
       "400644                  120             False         False            False   \n",
       "400645                  120             False         False            False   \n",
       "400646                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "400642                       Gas  \n",
       "400643                       Gas  \n",
       "400644                       Gas  \n",
       "400645                       Gas  \n",
       "400646                       Gas  \n",
       "\n",
       "[395404 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/KS/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/KS-day/2019-dec-day-KS.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>781ec8442b037999e55ca700ac584e460a0304d6</td>\n",
       "      <td>2019-12-13 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>622</td>\n",
       "      <td>678</td>\n",
       "      <td>637</td>\n",
       "      <td>KS</td>\n",
       "      <td>Salina</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2019-12-11 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>722</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>2019-12-07 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>682</td>\n",
       "      <td>732</td>\n",
       "      <td>682</td>\n",
       "      <td>KS</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>100</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5</td>\n",
       "      <td>2019-12-28 17:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>665</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Tooeka</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2019-12-16 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>657</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449661</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-12-24 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>678</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449662</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-12-31 13:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449663</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-12-03 10:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449664</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-12-05 08:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>664</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>449665</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2019-12-31 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>441028 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       781ec8442b037999e55ca700ac584e460a0304d6  2019-12-13 13:15:00 UTC   \n",
       "1       7f9688a0b01344c1f2804913546cad73007d0cf4  2019-12-11 18:15:00 UTC   \n",
       "2       fc006352113dcc3ae14023c9e52f72483f64b65e  2019-12-07 16:15:00 UTC   \n",
       "3       f7a1ab91ae0f0bad91b2b3dcb13d4fffbeaeb9b5  2019-12-28 17:05:00 UTC   \n",
       "4       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2019-12-16 16:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "449661  842c02deec3009c2808be94f10e56264890a12f4  2019-12-24 18:10:00 UTC   \n",
       "449662  842c02deec3009c2808be94f10e56264890a12f4  2019-12-31 13:30:00 UTC   \n",
       "449663  842c02deec3009c2808be94f10e56264890a12f4  2019-12-03 10:25:00 UTC   \n",
       "449664  842c02deec3009c2808be94f10e56264890a12f4  2019-12-05 08:00:00 UTC   \n",
       "449665  842c02deec3009c2808be94f10e56264890a12f4  2019-12-31 17:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               622                      678   \n",
       "1          heat          auto               722                      720   \n",
       "2          auto          hold               682                      732   \n",
       "3          heat          hold               665                      670   \n",
       "4          heat          auto               657                      660   \n",
       "...         ...           ...               ...                      ...   \n",
       "449661     heat          auto               678                      680   \n",
       "449662     heat          hold               696                      710   \n",
       "449663     heat          hold               658                      660   \n",
       "449664     heat          hold               664                      650   \n",
       "449665     heat          hold               683                      690   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           637            KS         Salina   \n",
       "1                           720            KS         Oswego   \n",
       "2                           682            KS  Arkansas City   \n",
       "3                           670            KS         Tooeka   \n",
       "4                           660            KS     Dodge City   \n",
       "...                         ...           ...            ...   \n",
       "449661                      680            KS       Scranton   \n",
       "449662                      710            KS       Scranton   \n",
       "449663                      660            KS       Scranton   \n",
       "449664                      650            KS       Scranton   \n",
       "449665                      690            KS       Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        67             False         False            False   \n",
       "1                       106             False         False            False   \n",
       "2                       100              True         False            False   \n",
       "3                        67             False         False            False   \n",
       "4                       105             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "449661                  120             False         False            False   \n",
       "449662                  120             False         False            False   \n",
       "449663                  120             False         False            False   \n",
       "449664                  120             False         False            False   \n",
       "449665                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "449661                       Gas  \n",
       "449662                       Gas  \n",
       "449663                       Gas  \n",
       "449664                       Gas  \n",
       "449665                       Gas  \n",
       "\n",
       "[441028 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/KS/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/KS-day/2020-dec-day-KS.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>e73d2071803a834c945f4eedd70daf1ac7d08fea</td>\n",
       "      <td>2020-12-20 17:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>705</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>KS</td>\n",
       "      <td>Paola</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>62a890bb783d07af9497a160b6a7f4ae41579c67</td>\n",
       "      <td>2020-12-26 18:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>646</td>\n",
       "      <td>750</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Murdock</td>\n",
       "      <td>118</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>db4a1cf416aa978037224279ee7abb5a7f3e57f1</td>\n",
       "      <td>2020-12-08 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>670</td>\n",
       "      <td>750</td>\n",
       "      <td>670</td>\n",
       "      <td>KS</td>\n",
       "      <td>Dodge City</td>\n",
       "      <td>105</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f9688a0b01344c1f2804913546cad73007d0cf4</td>\n",
       "      <td>2020-12-20 17:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>KS</td>\n",
       "      <td>Oswego</td>\n",
       "      <td>106</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7bf217daebd9144ce543a1754527180e81166ccf</td>\n",
       "      <td>2020-12-06 12:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>800</td>\n",
       "      <td>720</td>\n",
       "      <td>KS</td>\n",
       "      <td>El Dorado</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429117</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-12-02 11:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>700</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429118</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-12-08 10:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>663</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429119</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-12-07 14:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429120</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-12-07 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>677</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>429121</th>\n",
       "      <td>842c02deec3009c2808be94f10e56264890a12f4</td>\n",
       "      <td>2020-12-30 11:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>KS</td>\n",
       "      <td>Scranton</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>424835 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       e73d2071803a834c945f4eedd70daf1ac7d08fea  2020-12-20 17:15:00 UTC   \n",
       "1       62a890bb783d07af9497a160b6a7f4ae41579c67  2020-12-26 18:45:00 UTC   \n",
       "2       db4a1cf416aa978037224279ee7abb5a7f3e57f1  2020-12-08 18:05:00 UTC   \n",
       "3       7f9688a0b01344c1f2804913546cad73007d0cf4  2020-12-20 17:20:00 UTC   \n",
       "4       7bf217daebd9144ce543a1754527180e81166ccf  2020-12-06 12:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "429117  842c02deec3009c2808be94f10e56264890a12f4  2020-12-02 11:50:00 UTC   \n",
       "429118  842c02deec3009c2808be94f10e56264890a12f4  2020-12-08 10:30:00 UTC   \n",
       "429119  842c02deec3009c2808be94f10e56264890a12f4  2020-12-07 14:10:00 UTC   \n",
       "429120  842c02deec3009c2808be94f10e56264890a12f4  2020-12-07 16:35:00 UTC   \n",
       "429121  842c02deec3009c2808be94f10e56264890a12f4  2020-12-30 11:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          auto               705                      710   \n",
       "1          auto          hold               646                      750   \n",
       "2          auto          auto               670                      750   \n",
       "3          heat          auto               734                      730   \n",
       "4          auto          auto               713                      800   \n",
       "...         ...           ...               ...                      ...   \n",
       "429117     heat          auto               669                      700   \n",
       "429118     heat          auto               663                      650   \n",
       "429119     heat          hold               698                      680   \n",
       "429120     heat          hold               677                      680   \n",
       "429121     heat          hold               685                      690   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState        City  \\\n",
       "0                           710            KS       Paola   \n",
       "1                           650            KS     Murdock   \n",
       "2                           670            KS  Dodge City   \n",
       "3                           730            KS      Oswego   \n",
       "4                           720            KS   El Dorado   \n",
       "...                         ...           ...         ...   \n",
       "429117                      680            KS    Scranton   \n",
       "429118                      650            KS    Scranton   \n",
       "429119                      680            KS    Scranton   \n",
       "429120                      680            KS    Scranton   \n",
       "429121                      690            KS    Scranton   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        99             False         False            False   \n",
       "1                       118             False         False            False   \n",
       "2                       105             False         False            False   \n",
       "3                       106             False         False            False   \n",
       "4                        67             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "429117                  120             False         False            False   \n",
       "429118                  120             False         False            False   \n",
       "429119                  120             False         False            False   \n",
       "429120                  120             False         False            False   \n",
       "429121                  120             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "429117                       Gas  \n",
       "429118                       Gas  \n",
       "429119                       Gas  \n",
       "429120                       Gas  \n",
       "429121                       Gas  \n",
       "\n",
       "[424835 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/KS/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/KS/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>698.909091</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>631.895833</td>\n",
       "      <td>820.000000</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>699.307692</td>\n",
       "      <td>700.923077</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>694.353659</td>\n",
       "      <td>798.077475</td>\n",
       "      <td>693.259326</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>753.827957</td>\n",
       "      <td>750.473118</td>\n",
       "      <td>750.473118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>692</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>686.310160</td>\n",
       "      <td>736.684492</td>\n",
       "      <td>686.684492</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>693</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>704.208804</td>\n",
       "      <td>761.207675</td>\n",
       "      <td>709.655756</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>694</th>\n",
       "      <td>ff0c14deff346a8a3dc87a89fb1ab6632be73334</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tonganoxie</td>\n",
       "      <td>676.125000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>695</th>\n",
       "      <td>ff926605815f441236df0441160fb16f5dc55311</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Wichita</td>\n",
       "      <td>678.125000</td>\n",
       "      <td>785.555556</td>\n",
       "      <td>684.305556</td>\n",
       "      <td>28.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2727 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   Identifier Month  Year HvacMode  \\\n",
       "0    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   dec  2017     auto   \n",
       "1    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   dec  2017     heat   \n",
       "2    00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   dec  2017     heat   \n",
       "3    01a0d7a563a1755d37bb856a01a56e452deb2010   dec  2017     auto   \n",
       "4    025ec7c58b524de74d8986f4896917c31052701a   dec  2017     heat   \n",
       "..                                        ...   ...   ...      ...   \n",
       "691  fe779904e8eede7395997bd2762cd654983d363a   dec  2020     heat   \n",
       "692  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   dec  2020     auto   \n",
       "693  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   dec  2020     auto   \n",
       "694  ff0c14deff346a8a3dc87a89fb1ab6632be73334   dec  2020     heat   \n",
       "695  ff926605815f441236df0441160fb16f5dc55311   dec  2020     auto   \n",
       "\n",
       "    CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0            auto        Fairway            698.909091   \n",
       "1            auto        Fairway            631.895833   \n",
       "2            hold        Fairway            699.307692   \n",
       "3            hold         Olathe            694.353659   \n",
       "4            hold  Overland Park            680.000000   \n",
       "..            ...            ...                   ...   \n",
       "691          hold        Andover            753.827957   \n",
       "692          auto    Eastborough            686.310160   \n",
       "693          hold    Eastborough            704.208804   \n",
       "694          hold     Tonganoxie            676.125000   \n",
       "695          auto        Wichita            678.125000   \n",
       "\n",
       "     TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                     760.000000                   700.000000   \n",
       "1                     820.000000                   640.000000   \n",
       "2                     700.923077                   700.000000   \n",
       "3                     798.077475                   693.259326   \n",
       "4                     680.000000                   680.000000   \n",
       "..                           ...                          ...   \n",
       "691                   750.473118                   750.473118   \n",
       "692                   736.684492                   686.684492   \n",
       "693                   761.207675                   709.655756   \n",
       "694                   680.000000                   680.000000   \n",
       "695                   785.555556                   684.305556   \n",
       "\n",
       "     Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                   80.0             False         False            False  \n",
       "1                   80.0             False         False            False  \n",
       "2                   80.0             False         False            False  \n",
       "3                   30.0             False         False            False  \n",
       "4                    5.0             False         False             True  \n",
       "..                   ...               ...           ...              ...  \n",
       "691                  0.0             False         False            False  \n",
       "692                 90.0             False         False            False  \n",
       "693                 90.0             False         False            False  \n",
       "694                 10.0             False         False            False  \n",
       "695                 28.0             False         False            False  \n",
       "\n",
       "[2727 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/KS/dec/\" + file)\n",
    "    KS_dec = pd.concat([KS_dec, df])\n",
    "    \n",
    "KS_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_dec.to_csv(\"Scraper_Output/State_Month_Day/KS/KS_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/KS/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fairway</td>\n",
       "      <td>759.714286</td>\n",
       "      <td>746.342857</td>\n",
       "      <td>745.914286</td>\n",
       "      <td>80.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01a0d7a563a1755d37bb856a01a56e452deb2010</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>753.352660</td>\n",
       "      <td>761.052565</td>\n",
       "      <td>671.215992</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>025ec7c58b524de74d8986f4896917c31052701a</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>730.612368</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0448b121ebca96144ab5e5feb5ab88942e98de42</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Olathe</td>\n",
       "      <td>733.516129</td>\n",
       "      <td>742.516129</td>\n",
       "      <td>741.741935</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>050b44b61d052cae8d52e817cc0f2bb78bb91535</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Overland Park</td>\n",
       "      <td>728.400000</td>\n",
       "      <td>749.200000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>55.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3019</th>\n",
       "      <td>fc006352113dcc3ae14023c9e52f72483f64b65e</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Arkansas City</td>\n",
       "      <td>732.458333</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3020</th>\n",
       "      <td>fdf82bf64cb1a67c28f91aa7246724223df57851</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Atchison</td>\n",
       "      <td>705.988506</td>\n",
       "      <td>730.689655</td>\n",
       "      <td>680.517241</td>\n",
       "      <td>40.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3021</th>\n",
       "      <td>fe779904e8eede7395997bd2762cd654983d363a</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Andover</td>\n",
       "      <td>752.840278</td>\n",
       "      <td>748.888889</td>\n",
       "      <td>748.888889</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3022</th>\n",
       "      <td>fe84f839012ce8d6db47aa28b3b5d7bafb8ece60</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Eastborough</td>\n",
       "      <td>702.469314</td>\n",
       "      <td>696.205776</td>\n",
       "      <td>696.054152</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3023</th>\n",
       "      <td>ff0c14deff346a8a3dc87a89fb1ab6632be73334</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tonganoxie</td>\n",
       "      <td>654.805556</td>\n",
       "      <td>655.000000</td>\n",
       "      <td>605.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15982 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00d2d0e5e17f6dc9a19aa132916fdc6ddacb983e   aug  2017     cool   \n",
       "1     01a0d7a563a1755d37bb856a01a56e452deb2010   aug  2017     auto   \n",
       "2     025ec7c58b524de74d8986f4896917c31052701a   aug  2017     cool   \n",
       "3     0448b121ebca96144ab5e5feb5ab88942e98de42   aug  2017     cool   \n",
       "4     050b44b61d052cae8d52e817cc0f2bb78bb91535   aug  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3019  fc006352113dcc3ae14023c9e52f72483f64b65e   jun  2021     auto   \n",
       "3020  fdf82bf64cb1a67c28f91aa7246724223df57851   jun  2021     auto   \n",
       "3021  fe779904e8eede7395997bd2762cd654983d363a   jun  2021     cool   \n",
       "3022  fe84f839012ce8d6db47aa28b3b5d7bafb8ece60   jun  2021     cool   \n",
       "3023  ff0c14deff346a8a3dc87a89fb1ab6632be73334   jun  2021     auto   \n",
       "\n",
       "     CalendarEvent           City  Temperature_ctrl_ave  \\\n",
       "0             hold        Fairway            759.714286   \n",
       "1             hold         Olathe            753.352660   \n",
       "2             hold  Overland Park            730.612368   \n",
       "3             hold         Olathe            733.516129   \n",
       "4             auto  Overland Park            728.400000   \n",
       "...            ...            ...                   ...   \n",
       "3019          hold  Arkansas City            732.458333   \n",
       "3020          hold       Atchison            705.988506   \n",
       "3021          hold        Andover            752.840278   \n",
       "3022          hold    Eastborough            702.469314   \n",
       "3023          hold     Tonganoxie            654.805556   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      746.342857                   745.914286   \n",
       "1                      761.052565                   671.215992   \n",
       "2                      740.000000                   720.000000   \n",
       "3                      742.516129                   741.741935   \n",
       "4                      749.200000                   680.000000   \n",
       "...                           ...                          ...   \n",
       "3019                   730.000000                   670.000000   \n",
       "3020                   730.689655                   680.517241   \n",
       "3021                   748.888889                   748.888889   \n",
       "3022                   696.205776                   696.054152   \n",
       "3023                   655.000000                   605.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                    80.0             False         False            False  \n",
       "1                    30.0             False         False            False  \n",
       "2                     5.0             False         False             True  \n",
       "3                    10.0             False         False            False  \n",
       "4                    55.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "3019                100.0              True         False            False  \n",
       "3020                 40.0             False         False            False  \n",
       "3021                  0.0             False         False            False  \n",
       "3022                 90.0             False         False            False  \n",
       "3023                 10.0             False         False            False  \n",
       "\n",
       "[15982 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "KS_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/KS/\" + file)\n",
    "    KS_all = pd.concat([KS_all, df])\n",
    "    \n",
    "KS_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "KS_all.to_csv(\"Scraper_Output/State_Month_Day/KS_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['KS']\n",
      "Unique jan_2018: ['KS']\n",
      "Unique jan_2019: ['KS']\n",
      "Unique jan_2020: ['KS']\n",
      "Unique jan_2021: ['KS']\n",
      "Unique feb_2017: ['KS']\n",
      "Unique feb_2018: ['KS']\n",
      "Unique feb_2019: ['KS']\n",
      "Unique feb_2020: ['KS']\n",
      "Unique feb_2021: ['KS']\n",
      "Unique jun_2017: ['KS']\n",
      "Unique jun_2018: ['KS']\n",
      "Unique jun_2019: ['KS']\n",
      "Unique jun_2020: ['KS']\n",
      "Unique jun_2021: ['KS']\n",
      "Unique jul_2017: ['KS']\n",
      "Unique jul_2018: ['KS']\n",
      "Unique jul_2019: ['KS']\n",
      "Unique jul_2020: ['KS']\n",
      "Unique jul_2021: ['KS']\n",
      "Unique aug_2017: ['KS']\n",
      "Unique aug_2018: ['KS']\n",
      "Unique aug_2019: ['KS']\n",
      "Unique aug_2020: ['KS']\n",
      "Unique dec_2017: ['KS']\n",
      "Unique dec_2018: ['KS']\n",
      "Unique dec_2019: ['KS']\n",
      "Unique dec_2020: ['KS']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mKSe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}