DYD_FL-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/FL-day/2017-jan-day-FL.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f7010700c59175433a883eba80cfcf5fa2edad2f</td>\n",
       "      <td>2017-01-22 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>800</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Newberry</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fbc4bb2d5762500329e74ad73cf523fc50df7f3d</td>\n",
       "      <td>2017-01-30 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e20a3e765cdf2b7a2626a87af28bd4f0b6587df8</td>\n",
       "      <td>2017-01-01 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>770</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>South Pasadena</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ade94a0e1eb1ff8d488cb6e6036362a8d2391975</td>\n",
       "      <td>2017-01-28 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>730</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Riverview</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ddda85d74ec02ede30a53c68043b86a3e0ee52be</td>\n",
       "      <td>2017-01-14 18:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Hialeah</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915550</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-01-01 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>627</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915551</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-01-13 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>657</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915552</th>\n",
       "      <td>430c429dc9875cf26ef69755a5b31c4b60e4ea75</td>\n",
       "      <td>2017-01-01 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>800</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915553</th>\n",
       "      <td>430c429dc9875cf26ef69755a5b31c4b60e4ea75</td>\n",
       "      <td>2017-01-10 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>665</td>\n",
       "      <td>800</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>915554</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-01-02 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>683</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>873750 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       f7010700c59175433a883eba80cfcf5fa2edad2f  2017-01-22 17:15:00 UTC   \n",
       "1       fbc4bb2d5762500329e74ad73cf523fc50df7f3d  2017-01-30 18:55:00 UTC   \n",
       "2       e20a3e765cdf2b7a2626a87af28bd4f0b6587df8  2017-01-01 12:35:00 UTC   \n",
       "3       ade94a0e1eb1ff8d488cb6e6036362a8d2391975  2017-01-28 17:45:00 UTC   \n",
       "4       ddda85d74ec02ede30a53c68043b86a3e0ee52be  2017-01-14 18:20:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "915550  805b979c88c6beb513fb8be8963da99c2cb05798  2017-01-01 14:00:00 UTC   \n",
       "915551  805b979c88c6beb513fb8be8963da99c2cb05798  2017-01-13 16:45:00 UTC   \n",
       "915552  430c429dc9875cf26ef69755a5b31c4b60e4ea75  2017-01-01 19:05:00 UTC   \n",
       "915553  430c429dc9875cf26ef69755a5b31c4b60e4ea75  2017-01-10 12:25:00 UTC   \n",
       "915554  805b979c88c6beb513fb8be8963da99c2cb05798  2017-01-02 12:25:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          auto          hold               748                      800   \n",
       "1          auto          auto               720                      760   \n",
       "2          auto          hold               725                      770   \n",
       "3          auto          hold               680                      730   \n",
       "4          auto          hold               755                      760   \n",
       "...         ...           ...               ...                      ...   \n",
       "915550     auto          auto               627                      800   \n",
       "915551     auto          auto               657                      800   \n",
       "915552     auto          hold               748                      800   \n",
       "915553     auto          hold               665                      800   \n",
       "915554     auto          auto               683                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                           750            FL             Newberry   \n",
       "1                           700            FL             Orlando    \n",
       "2                           720            FL       South Pasadena   \n",
       "3                           650            FL            Riverview   \n",
       "4                           710            FL              Hialeah   \n",
       "...                         ...           ...                  ...   \n",
       "915550                      640            FL  Green Cover Springs   \n",
       "915551                      640            FL  Green Cover Springs   \n",
       "915552                      650            FL  Green Cover Springs   \n",
       "915553                      650            FL  Green Cover Springs   \n",
       "915554                      640            FL  Green Cover Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        10             False         False             True   \n",
       "1                        15             False         False             True   \n",
       "2                        50              True         False            False   \n",
       "3                         5             False         False             True   \n",
       "4                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "915550                    5              True         False             True   \n",
       "915551                    5              True         False             True   \n",
       "915552                    5              True         False             True   \n",
       "915553                    5              True         False             True   \n",
       "915554                    5              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                       Electric  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "915550                  Electric  \n",
       "915551                  Electric  \n",
       "915552                  Electric  \n",
       "915553                  Electric  \n",
       "915554                  Electric  \n",
       "\n",
       "[873750 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00020f16d85e6d8c70fc67725f03ab06b116fead</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Jacksonville</th>\n",
       "      <td>713.266667</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0024f0f900a4ff6923e95a2a20ee94c699ecf167</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">cool</th>\n",
       "      <th>auto</th>\n",
       "      <th>Miami</th>\n",
       "      <td>742.527842</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>780.939675</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Miami</th>\n",
       "      <td>710.424242</td>\n",
       "      <td>754.848485</td>\n",
       "      <td>754.848485</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">01047c4cced05cfd81b6a7c9c263239317203975</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Oviedo</th>\n",
       "      <td>743.220588</td>\n",
       "      <td>765.000000</td>\n",
       "      <td>715.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cool</th>\n",
       "      <th>auto</th>\n",
       "      <th>Oviedo</th>\n",
       "      <td>746.829452</td>\n",
       "      <td>760.395943</td>\n",
       "      <td>713.435011</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fed8bcb67d870524eabe5fe9d83721a808419d6e</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Sebastian</th>\n",
       "      <td>731.666667</td>\n",
       "      <td>739.416667</td>\n",
       "      <td>739.416667</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Sebastian</th>\n",
       "      <td>726.666667</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">feea0a3dc64015ecc77007c21d048b31346d3261</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Sunrise</th>\n",
       "      <td>753.476190</td>\n",
       "      <td>773.909254</td>\n",
       "      <td>682.776280</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Sunrise</th>\n",
       "      <td>746.239939</td>\n",
       "      <td>793.655120</td>\n",
       "      <td>688.470199</td>\n",
       "      <td>6.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>cool</th>\n",
       "      <th>hold</th>\n",
       "      <th>Oakland Park</th>\n",
       "      <td>732.307692</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1963 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                         Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                 \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville            713.266667   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                   742.527842   \n",
       "                                                             hold          Miami                   710.424242   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                  743.220588   \n",
       "                                                    cool     auto          Oviedo                  746.829452   \n",
       "...                                                                                                       ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian               731.666667   \n",
       "                                                             hold          Sebastian               726.666667   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                 753.476190   \n",
       "                                                             hold          Sunrise                 746.239939   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park            732.307692   \n",
       "\n",
       "                                                                                         TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville                   760.000000   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                          800.000000   \n",
       "                                                             hold          Miami                          754.848485   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                         765.000000   \n",
       "                                                    cool     auto          Oviedo                         760.395943   \n",
       "...                                                                                                              ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian                      739.416667   \n",
       "                                                             hold          Sebastian                      720.000000   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                        773.909254   \n",
       "                                                             hold          Sunrise                        793.655120   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park                   730.000000   \n",
       "\n",
       "                                                                                         TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                        \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville                   670.000000   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                          780.939675   \n",
       "                                                             hold          Miami                          754.848485   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                         715.000000   \n",
       "                                                    cool     auto          Oviedo                         713.435011   \n",
       "...                                                                                                              ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian                      739.416667   \n",
       "                                                             hold          Sebastian                      720.000000   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                        682.776280   \n",
       "                                                             hold          Sunrise                        688.470199   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park                   730.000000   \n",
       "\n",
       "                                                                                         Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville                  5.0   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                         0.0   \n",
       "                                                             hold          Miami                         0.0   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                        0.0   \n",
       "                                                    cool     auto          Oviedo                        0.0   \n",
       "...                                                                                                      ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian                    25.0   \n",
       "                                                             hold          Sebastian                    25.0   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                       6.0   \n",
       "                                                             hold          Sunrise                       6.0   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park                 30.0   \n",
       "\n",
       "                                                                                         allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                             \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville             False   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                    False   \n",
       "                                                             hold          Miami                    False   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                   False   \n",
       "                                                    cool     auto          Oviedo                   False   \n",
       "...                                                                                                   ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian                False   \n",
       "                                                             hold          Sebastian                False   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                  False   \n",
       "                                                             hold          Sunrise                  False   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park             False   \n",
       "\n",
       "                                                                                         Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                         \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville         False   \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                False   \n",
       "                                                             hold          Miami                False   \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo               False   \n",
       "                                                    cool     auto          Oviedo               False   \n",
       "...                                                                                               ...   \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian            False   \n",
       "                                                             hold          Sebastian            False   \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise              False   \n",
       "                                                             hold          Sunrise              False   \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park         False   \n",
       "\n",
       "                                                                                         Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                           \n",
       "00020f16d85e6d8c70fc67725f03ab06b116fead Jan   2017 auto     auto          Jacksonville             True  \n",
       "0024f0f900a4ff6923e95a2a20ee94c699ecf167 Jan   2017 cool     auto          Miami                   False  \n",
       "                                                             hold          Miami                   False  \n",
       "01047c4cced05cfd81b6a7c9c263239317203975 Jan   2017 auto     auto          Oviedo                   True  \n",
       "                                                    cool     auto          Oviedo                   True  \n",
       "...                                                                                                  ...  \n",
       "fed8bcb67d870524eabe5fe9d83721a808419d6e Jan   2017 heat     auto          Sebastian               False  \n",
       "                                                             hold          Sebastian               False  \n",
       "feea0a3dc64015ecc77007c21d048b31346d3261 Jan   2017 auto     auto          Sunrise                 False  \n",
       "                                                             hold          Sunrise                 False  \n",
       "ffcf942aa8736eaa25440e6817dea4d9efc3c67f Jan   2017 cool     hold          Oakland Park            False  \n",
       "\n",
       "[1963 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/FL/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/FL-day/2018-jan-day-FL.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6b47f19e2ac3de5a5c088d2941ae61bb91f316b0</td>\n",
       "      <td>2018-01-06 15:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>709</td>\n",
       "      <td>698</td>\n",
       "      <td>FL</td>\n",
       "      <td>St Petersburg</td>\n",
       "      <td>55</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6858313966d2545da544ff439290f5bf27f4fa40</td>\n",
       "      <td>2018-01-11 14:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>795</td>\n",
       "      <td>725</td>\n",
       "      <td>FL</td>\n",
       "      <td>New Smyrna Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c9b4a1b8b7fddea51aa62557d1e0077f59c114dc</td>\n",
       "      <td>2018-01-03 07:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>FL</td>\n",
       "      <td>-</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0d0fad2a370ee198dc9656c834c57253c8fa1a2f</td>\n",
       "      <td>2018-01-26 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>752</td>\n",
       "      <td>690</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami Beach</td>\n",
       "      <td>79</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>e478562afbe7db9cd8aa42235cb8fb37cc134b92</td>\n",
       "      <td>2018-01-07 16:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>695</td>\n",
       "      <td>695</td>\n",
       "      <td>FL</td>\n",
       "      <td>wilton manors</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713878</th>\n",
       "      <td>cfbff081e413476780f609e5c7dcf2020c0677b3</td>\n",
       "      <td>2018-01-19 15:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>FL</td>\n",
       "      <td>Dania Beach</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713879</th>\n",
       "      <td>0f010f452f9f5edaac659b79ea2e3c9701813846</td>\n",
       "      <td>2018-01-05 18:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>765</td>\n",
       "      <td>765</td>\n",
       "      <td>FL</td>\n",
       "      <td>Longwood</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713880</th>\n",
       "      <td>f1c94a3ce3513081ef44beb8de5e0e10f522f917</td>\n",
       "      <td>2018-01-24 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>FL</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>26</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713881</th>\n",
       "      <td>db1f63574972d52866f2b1c108a4b1f6735886d2</td>\n",
       "      <td>2018-01-17 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pembroke pines</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2713882</th>\n",
       "      <td>a8b12ab27e5fd886a42b40f7b865429b895ef384</td>\n",
       "      <td>2018-01-19 09:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>691</td>\n",
       "      <td>765</td>\n",
       "      <td>695</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Park</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2591443 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        6b47f19e2ac3de5a5c088d2941ae61bb91f316b0  2018-01-06 15:10:00 UTC   \n",
       "2        6858313966d2545da544ff439290f5bf27f4fa40  2018-01-11 14:45:00 UTC   \n",
       "3        c9b4a1b8b7fddea51aa62557d1e0077f59c114dc  2018-01-03 07:05:00 UTC   \n",
       "4        0d0fad2a370ee198dc9656c834c57253c8fa1a2f  2018-01-26 15:35:00 UTC   \n",
       "5        e478562afbe7db9cd8aa42235cb8fb37cc134b92  2018-01-07 16:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2713878  cfbff081e413476780f609e5c7dcf2020c0677b3  2018-01-19 15:25:00 UTC   \n",
       "2713879  0f010f452f9f5edaac659b79ea2e3c9701813846  2018-01-05 18:10:00 UTC   \n",
       "2713880  f1c94a3ce3513081ef44beb8de5e0e10f522f917  2018-01-24 12:30:00 UTC   \n",
       "2713881  db1f63574972d52866f2b1c108a4b1f6735886d2  2018-01-17 15:00:00 UTC   \n",
       "2713882  a8b12ab27e5fd886a42b40f7b865429b895ef384  2018-01-19 09:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               673                      709   \n",
       "2           auto          hold               735                      795   \n",
       "3           heat          hold               693                      695   \n",
       "4           cool          auto               734                      752   \n",
       "5           heat          hold               714                      695   \n",
       "...          ...           ...               ...                      ...   \n",
       "2713878     auto          hold               713                      765   \n",
       "2713879     heat          hold               766                      765   \n",
       "2713880     auto          auto               692                      765   \n",
       "2713881     auto          hold               718                      765   \n",
       "2713882     auto          hold               691                      765   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            698            FL     St Petersburg   \n",
       "2                            725            FL  New Smyrna Beach   \n",
       "3                            695            FL                 -   \n",
       "4                            690            FL       Miami Beach   \n",
       "5                            695            FL    wilton manors    \n",
       "...                          ...           ...               ...   \n",
       "2713878                      715            FL       Dania Beach   \n",
       "2713879                      765            FL          Longwood   \n",
       "2713880                      715            FL      Jacksonville   \n",
       "2713881                      715            FL    Pembroke pines   \n",
       "2713882                      695            FL       Winter Park   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         55              True         False             True   \n",
       "2                         10              True         False             True   \n",
       "3                         17             False         False             True   \n",
       "4                         79             False         False            False   \n",
       "5                          9              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2713878                   10             False         False            False   \n",
       "2713879                   40              True         False             True   \n",
       "2713880                   26             False         False             True   \n",
       "2713881                   50             False         False            False   \n",
       "2713882                   60             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2713878                       Gas  \n",
       "2713879                  Electric  \n",
       "2713880                  Electric  \n",
       "2713881                       Gas  \n",
       "2713882                       Gas  \n",
       "\n",
       "[2591443 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/FL/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/FL-day/2019-jan-day-FL.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>531ef970345a9647e7bbe147e8de34e458f7c47b</td>\n",
       "      <td>2019-01-23 13:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Valrico</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4053afa5c9936c8c8ba088d44f87bfc2fb25bbcc</td>\n",
       "      <td>2019-01-17 16:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>684</td>\n",
       "      <td>720</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lutz</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c1d5949ab8e0c06eb99b45864ee18c529b3b9ccd</td>\n",
       "      <td>2019-01-19 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>740</td>\n",
       "      <td>690</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7173c493c97ba3c84ef17be09fc2d2a14a137228</td>\n",
       "      <td>2019-01-19 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>658</td>\n",
       "      <td>750</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>56b9dae26f97a5ca2de4f82cb113fb73ecc0e036</td>\n",
       "      <td>2019-01-28 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>653</td>\n",
       "      <td>820</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217117</th>\n",
       "      <td>78c544d449e0387edcff821adcda02ed6712c8ea</td>\n",
       "      <td>2019-01-11 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Venice</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217118</th>\n",
       "      <td>fbb4175d69b11497cf8db1965cd6947ffd9e659a</td>\n",
       "      <td>2019-01-28 12:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>770</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>West Palm Beach</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217119</th>\n",
       "      <td>6b55ca21b381b89f7da9b98d933d4a31636e4769</td>\n",
       "      <td>2019-01-27 16:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Navarre</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217120</th>\n",
       "      <td>4d0b2b869cd1299c830b5fd3b09f9a3aa42da99c</td>\n",
       "      <td>2019-01-13 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>790</td>\n",
       "      <td>690</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tamarac</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4217121</th>\n",
       "      <td>6fd58627d589a05e80493a4ad2f39edca2c02d21</td>\n",
       "      <td>2019-01-19 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lake Mary</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4013362 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        531ef970345a9647e7bbe147e8de34e458f7c47b  2019-01-23 13:50:00 UTC   \n",
       "1        4053afa5c9936c8c8ba088d44f87bfc2fb25bbcc  2019-01-17 16:25:00 UTC   \n",
       "2        c1d5949ab8e0c06eb99b45864ee18c529b3b9ccd  2019-01-19 13:45:00 UTC   \n",
       "3        7173c493c97ba3c84ef17be09fc2d2a14a137228  2019-01-19 15:15:00 UTC   \n",
       "4        56b9dae26f97a5ca2de4f82cb113fb73ecc0e036  2019-01-28 13:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4217117  78c544d449e0387edcff821adcda02ed6712c8ea  2019-01-11 18:25:00 UTC   \n",
       "4217118  fbb4175d69b11497cf8db1965cd6947ffd9e659a  2019-01-28 12:05:00 UTC   \n",
       "4217119  6b55ca21b381b89f7da9b98d933d4a31636e4769  2019-01-27 16:50:00 UTC   \n",
       "4217120  4d0b2b869cd1299c830b5fd3b09f9a3aa42da99c  2019-01-13 16:20:00 UTC   \n",
       "4217121  6fd58627d589a05e80493a4ad2f39edca2c02d21  2019-01-19 18:20:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          auto               756                      760   \n",
       "1           auto          auto               684                      720   \n",
       "2           auto          hold               703                      740   \n",
       "3           auto          hold               658                      750   \n",
       "4           cool          auto               653                      820   \n",
       "...          ...           ...               ...                      ...   \n",
       "4217117     cool          hold               668                      760   \n",
       "4217118     auto          auto               718                      770   \n",
       "4217119     heat          auto               696                      720   \n",
       "4217120     cool          hold               765                      790   \n",
       "4217121     cool          hold               706                      700   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            760            FL          Valrico   \n",
       "1                            670            FL             Lutz   \n",
       "2                            690            FL          Orlando   \n",
       "3                            650            FL          Orlando   \n",
       "4                            650            FL        Kissimmee   \n",
       "...                          ...           ...              ...   \n",
       "4217117                      680            FL           Venice   \n",
       "4217118                      720            FL  West Palm Beach   \n",
       "4217119                      720            FL          Navarre   \n",
       "4217120                      690            FL          Tamarac   \n",
       "4217121                      700            FL        Lake Mary   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False             True   \n",
       "1                          0              True         False             True   \n",
       "2                          8             False         False             True   \n",
       "3                         50             False         False             True   \n",
       "4                          5             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "4217117                   20              True         False             True   \n",
       "4217118                   15             False         False            False   \n",
       "4217119                    0             False         False             True   \n",
       "4217120                    0             False         False            False   \n",
       "4217121                   10             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "4217117                  Electric  \n",
       "4217118                       Gas  \n",
       "4217119                  Electric  \n",
       "4217120                       Gas  \n",
       "4217121                       Gas  \n",
       "\n",
       "[4013362 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/FL/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/FL-day/2020-jan-day-FL.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3b09c6048ef138b5049f3ee160c5e9254dda559e</td>\n",
       "      <td>2020-01-14 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pinellas Park</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f87f3bd99a20509a3bc203c71cf6e3ffd1cf76f6</td>\n",
       "      <td>2020-01-26 16:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>FL</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fc80700b84027c9e1b092d52b83a5dd558f2e33f</td>\n",
       "      <td>2020-01-01 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>destin</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32ec01b99b2b667fd1ed548a98ce4c0edc5cff38</td>\n",
       "      <td>2020-01-14 15:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>730</td>\n",
       "      <td>702</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Springs</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>159a3848a6de3d32fa9fcb91bb122a1e1a5d6749</td>\n",
       "      <td>2020-01-18 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Atlantis</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4425109</th>\n",
       "      <td>a708992f20f4f7765e4ae0300ed2c3514a6a2444</td>\n",
       "      <td>2020-01-19 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>742</td>\n",
       "      <td>742</td>\n",
       "      <td>FL</td>\n",
       "      <td>Boca Raton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4425110</th>\n",
       "      <td>c30325a73b93609a409be5b926382b085930d958</td>\n",
       "      <td>2020-01-05 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lake Mary</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4425111</th>\n",
       "      <td>4a5663d80ac77cbb9922275581c4c44696a6fcd7</td>\n",
       "      <td>2020-01-07 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>693</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Melbourne</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4425112</th>\n",
       "      <td>e949cb54d57ee86d897c0a539f25756e5d2dd50f</td>\n",
       "      <td>2020-01-03 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>DeBary</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4425115</th>\n",
       "      <td>78c544d449e0387edcff821adcda02ed6712c8ea</td>\n",
       "      <td>2020-01-14 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Venice</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4238127 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        3b09c6048ef138b5049f3ee160c5e9254dda559e  2020-01-14 19:05:00 UTC   \n",
       "1        f87f3bd99a20509a3bc203c71cf6e3ffd1cf76f6  2020-01-26 16:10:00 UTC   \n",
       "2        fc80700b84027c9e1b092d52b83a5dd558f2e33f  2020-01-01 14:50:00 UTC   \n",
       "3        32ec01b99b2b667fd1ed548a98ce4c0edc5cff38  2020-01-14 15:00:00 UTC   \n",
       "4        159a3848a6de3d32fa9fcb91bb122a1e1a5d6749  2020-01-18 14:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4425109  a708992f20f4f7765e4ae0300ed2c3514a6a2444  2020-01-19 16:15:00 UTC   \n",
       "4425110  c30325a73b93609a409be5b926382b085930d958  2020-01-05 13:10:00 UTC   \n",
       "4425111  4a5663d80ac77cbb9922275581c4c44696a6fcd7  2020-01-07 15:45:00 UTC   \n",
       "4425112  e949cb54d57ee86d897c0a539f25756e5d2dd50f  2020-01-03 14:05:00 UTC   \n",
       "4425115  78c544d449e0387edcff821adcda02ed6712c8ea  2020-01-14 15:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               767                      760   \n",
       "1           auto          auto               708                      725   \n",
       "2           heat          hold               710                      720   \n",
       "3           auto          hold               718                      730   \n",
       "4           cool          auto               712                      710   \n",
       "...          ...           ...               ...                      ...   \n",
       "4425109     cool          hold               740                      742   \n",
       "4425110     heat          hold               697                      700   \n",
       "4425111     cool          auto               693                      710   \n",
       "4425112     auto          auto               732                      740   \n",
       "4425115     cool          auto               757                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            710            FL  Pinellas Park   \n",
       "1                            675            FL      Homestead   \n",
       "2                            720            FL         destin   \n",
       "3                            702            FL  Coral Springs   \n",
       "4                            710            FL       Atlantis   \n",
       "...                          ...           ...            ...   \n",
       "4425109                      742            FL     Boca Raton   \n",
       "4425110                      700            FL      Lake Mary   \n",
       "4425111                      710            FL      Melbourne   \n",
       "4425112                      680            FL         DeBary   \n",
       "4425115                      760            FL         Venice   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False             True   \n",
       "1                         10             False         False             True   \n",
       "2                         30             False         False             True   \n",
       "3                         20             False         False            False   \n",
       "4                         40             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "4425109                    0             False         False            False   \n",
       "4425110                    5             False         False             True   \n",
       "4425111                   25             False         False            False   \n",
       "4425112                   10              True         False             True   \n",
       "4425115                   20              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "4425109                       Gas  \n",
       "4425110                  Electric  \n",
       "4425111                       Gas  \n",
       "4425112                  Electric  \n",
       "4425115                  Electric  \n",
       "\n",
       "[4238127 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/FL/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/FL-day/2021-jan-day-FL.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9c2eaee56dd3dc278aadd1b505f80555e34aabf7</td>\n",
       "      <td>2021-01-03 12:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>39bbeab58bc1578fb593378a6fc8a1c2e37a2ae6</td>\n",
       "      <td>2021-01-20 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>775</td>\n",
       "      <td>645</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ee1aa1f9de7208a24ef1d093c9cd33fc8aed8193</td>\n",
       "      <td>2021-01-09 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palmetto</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e50c55bfd0e580222d9d0bac58312254a443f7ab</td>\n",
       "      <td>2021-01-02 09:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>703</td>\n",
       "      <td>703</td>\n",
       "      <td>FL</td>\n",
       "      <td>Panama City Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dedce87a13dc4cceeb5cded9a4410d594523ac72</td>\n",
       "      <td>2021-01-28 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2541669</th>\n",
       "      <td>a9e0c1277d7c419c9c53a177622342ee019c3418</td>\n",
       "      <td>2021-01-25 18:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>799</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Cloud</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2541670</th>\n",
       "      <td>e1f04e7283ec625fd77870c409b6849ff6a65b73</td>\n",
       "      <td>2021-01-24 12:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Petersburg</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2541671</th>\n",
       "      <td>522e8de47bb38ecea118c6a329ed93dda0aae3dc</td>\n",
       "      <td>2021-01-22 18:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Apollo Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2541672</th>\n",
       "      <td>1d60ecf4c1cbf82437bbb548de8417f9e7a853c4</td>\n",
       "      <td>2021-01-21 16:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2541673</th>\n",
       "      <td>85a4410021a911b489cb01c5c151e4fa4793f3f5</td>\n",
       "      <td>2021-01-02 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Delray Beach</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2423623 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        9c2eaee56dd3dc278aadd1b505f80555e34aabf7  2021-01-03 12:25:00 UTC   \n",
       "1        39bbeab58bc1578fb593378a6fc8a1c2e37a2ae6  2021-01-20 17:20:00 UTC   \n",
       "2        ee1aa1f9de7208a24ef1d093c9cd33fc8aed8193  2021-01-09 17:40:00 UTC   \n",
       "3        e50c55bfd0e580222d9d0bac58312254a443f7ab  2021-01-02 09:10:00 UTC   \n",
       "4        dedce87a13dc4cceeb5cded9a4410d594523ac72  2021-01-28 18:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2541669  a9e0c1277d7c419c9c53a177622342ee019c3418  2021-01-25 18:00:00 UTC   \n",
       "2541670  e1f04e7283ec625fd77870c409b6849ff6a65b73  2021-01-24 12:30:00 UTC   \n",
       "2541671  522e8de47bb38ecea118c6a329ed93dda0aae3dc  2021-01-22 18:10:00 UTC   \n",
       "2541672  1d60ecf4c1cbf82437bbb548de8417f9e7a853c4  2021-01-21 16:00:00 UTC   \n",
       "2541673  85a4410021a911b489cb01c5c151e4fa4793f3f5  2021-01-02 13:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               756                      757   \n",
       "1           auto          hold               731                      775   \n",
       "2           cool          hold               734                      755   \n",
       "3           cool          hold               645                      703   \n",
       "4           cool          hold               697                      697   \n",
       "...          ...           ...               ...                      ...   \n",
       "2541669     heat          hold               799                      770   \n",
       "2541670     cool          hold               719                      760   \n",
       "2541671     cool          hold               716                      760   \n",
       "2541672     cool          hold               689                      760   \n",
       "2541673     cool          hold               760                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            757            FL       Oakland Park   \n",
       "1                            645            FL         Wellington   \n",
       "2                            755            FL           Palmetto   \n",
       "3                            703            FL  Panama City Beach   \n",
       "4                            697            FL              Miami   \n",
       "...                          ...           ...                ...   \n",
       "2541669                      760            FL        Saint Cloud   \n",
       "2541670                      760            FL   Saint Petersburg   \n",
       "2541671                      760            FL       Apollo Beach   \n",
       "2541672                      760            FL            Orlando   \n",
       "2541673                      760            FL       Delray Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False            False   \n",
       "1                         30             False         False            False   \n",
       "2                          5              True         False            False   \n",
       "3                          0             False         False             True   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2541669                    5             False         False             True   \n",
       "2541670                   77             False         False             True   \n",
       "2541671                    0              True         False            False   \n",
       "2541672                    0              True         False             True   \n",
       "2541673                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2541669                  Electric  \n",
       "2541670                  Electric  \n",
       "2541671                       Gas  \n",
       "2541672                  Electric  \n",
       "2541673                       Gas  \n",
       "\n",
       "[2423623 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/FL/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>713.266667</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>742.527842</td>\n",
       "      <td>800.000000</td>\n",
       "      <td>780.939675</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>710.424242</td>\n",
       "      <td>754.848485</td>\n",
       "      <td>754.848485</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01047c4cced05cfd81b6a7c9c263239317203975</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>743.220588</td>\n",
       "      <td>765.000000</td>\n",
       "      <td>715.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01047c4cced05cfd81b6a7c9c263239317203975</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>746.829452</td>\n",
       "      <td>760.395943</td>\n",
       "      <td>713.435011</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3897</th>\n",
       "      <td>ffa4468a31a8e35dd633f0022ea0b855501c5ba9</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>726.165608</td>\n",
       "      <td>725.971416</td>\n",
       "      <td>724.547641</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3898</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>721.237500</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>699.350000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3899</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>719.427136</td>\n",
       "      <td>717.492462</td>\n",
       "      <td>716.552764</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3900</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>753.402299</td>\n",
       "      <td>775.126437</td>\n",
       "      <td>710.896552</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3901</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>734.365079</td>\n",
       "      <td>730.988095</td>\n",
       "      <td>730.277778</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>26251 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   Jan  2017     auto   \n",
       "1     0024f0f900a4ff6923e95a2a20ee94c699ecf167   Jan  2017     cool   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   Jan  2017     cool   \n",
       "3     01047c4cced05cfd81b6a7c9c263239317203975   Jan  2017     auto   \n",
       "4     01047c4cced05cfd81b6a7c9c263239317203975   Jan  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3897  ffa4468a31a8e35dd633f0022ea0b855501c5ba9   Jan  2021     heat   \n",
       "3898  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   Jan  2021     auto   \n",
       "3899  fffcd8322781a8483039a10c259725e9da9cfdec   Jan  2021     cool   \n",
       "3900  ffff46e404a9831c28d8be1a3e427a4377690c6b   Jan  2021     auto   \n",
       "3901  ffff46e404a9831c28d8be1a3e427a4377690c6b   Jan  2021     heat   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            713.266667   \n",
       "1             auto         Miami            742.527842   \n",
       "2             hold         Miami            710.424242   \n",
       "3             auto        Oviedo            743.220588   \n",
       "4             auto        Oviedo            746.829452   \n",
       "...            ...           ...                   ...   \n",
       "3897          hold    Palm Coast            726.165608   \n",
       "3898          hold  Oakland Park            721.237500   \n",
       "3899          hold   North Miami            719.427136   \n",
       "3900          hold       Parrish            753.402299   \n",
       "3901          hold       Parrish            734.365079   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      760.000000                   670.000000   \n",
       "1                      800.000000                   780.939675   \n",
       "2                      754.848485                   754.848485   \n",
       "3                      765.000000                   715.000000   \n",
       "4                      760.395943                   713.435011   \n",
       "...                           ...                          ...   \n",
       "3897                   725.971416                   724.547641   \n",
       "3898                   760.000000                   699.350000   \n",
       "3899                   717.492462                   716.552764   \n",
       "3900                   775.126437                   710.896552   \n",
       "3901                   730.988095                   730.277778   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     0.0             False         False             True  \n",
       "4                     0.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "3897                 10.0             False         False             True  \n",
       "3898                 30.0             False         False            False  \n",
       "3899                 70.0             False         False            False  \n",
       "3900                  0.0             False         False             True  \n",
       "3901                  0.0             False         False             True  \n",
       "\n",
       "[26251 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/jan/\" + file)\n",
    "    FL_jan = pd.concat([FL_jan, df])\n",
    "    \n",
    "FL_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_jan.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/FL-day/2017-feb-day-FL.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7e690786f4d499ba5a182bc9bf5f8299b40fd99c</td>\n",
       "      <td>2017-02-26 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coconut Creek</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>754714343fa0ebaf6d54291c29f4b445680d4a73</td>\n",
       "      <td>2017-02-04 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>feea0a3dc64015ecc77007c21d048b31346d3261</td>\n",
       "      <td>2017-02-14 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>773</td>\n",
       "      <td>760</td>\n",
       "      <td>630</td>\n",
       "      <td>FL</td>\n",
       "      <td>Sunrise</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feea0a3dc64015ecc77007c21d048b31346d3261</td>\n",
       "      <td>2017-02-05 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>780</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Sunrise</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ad6110c12ed85aeff51dd158dfe57c48bd9e9b8c</td>\n",
       "      <td>2017-02-12 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827807</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-02-14 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>633</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827808</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-02-26 18:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>676</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827809</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-02-11 12:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>624</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827810</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2017-02-02 14:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>635</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>827811</th>\n",
       "      <td>430c429dc9875cf26ef69755a5b31c4b60e4ea75</td>\n",
       "      <td>2017-02-13 11:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>800</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>786123 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       7e690786f4d499ba5a182bc9bf5f8299b40fd99c  2017-02-26 13:35:00 UTC   \n",
       "2       754714343fa0ebaf6d54291c29f4b445680d4a73  2017-02-04 12:45:00 UTC   \n",
       "3       feea0a3dc64015ecc77007c21d048b31346d3261  2017-02-14 18:50:00 UTC   \n",
       "4       feea0a3dc64015ecc77007c21d048b31346d3261  2017-02-05 18:05:00 UTC   \n",
       "5       ad6110c12ed85aeff51dd158dfe57c48bd9e9b8c  2017-02-12 17:15:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "827807  805b979c88c6beb513fb8be8963da99c2cb05798  2017-02-14 13:50:00 UTC   \n",
       "827808  805b979c88c6beb513fb8be8963da99c2cb05798  2017-02-26 18:50:00 UTC   \n",
       "827809  805b979c88c6beb513fb8be8963da99c2cb05798  2017-02-11 12:00:00 UTC   \n",
       "827810  805b979c88c6beb513fb8be8963da99c2cb05798  2017-02-02 14:05:00 UTC   \n",
       "827811  430c429dc9875cf26ef69755a5b31c4b60e4ea75  2017-02-13 11:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               770                      770   \n",
       "2          cool          auto               733                      760   \n",
       "3          auto          auto               773                      760   \n",
       "4          auto          auto               751                      780   \n",
       "5          cool          hold               732                      730   \n",
       "...         ...           ...               ...                      ...   \n",
       "827807     auto          auto               633                      800   \n",
       "827808     auto          auto               676                      800   \n",
       "827809     auto          auto               624                      800   \n",
       "827810     auto          auto               635                      800   \n",
       "827811     auto          hold               728                      800   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                           770            FL        Coconut Creek   \n",
       "2                           670            FL            Kissimmee   \n",
       "3                           630            FL              Sunrise   \n",
       "4                           670            FL              Sunrise   \n",
       "5                           730            FL           Wellington   \n",
       "...                         ...           ...                  ...   \n",
       "827807                      640            FL  Green Cover Springs   \n",
       "827808                      640            FL  Green Cover Springs   \n",
       "827809                      640            FL  Green Cover Springs   \n",
       "827810                      640            FL  Green Cover Springs   \n",
       "827811                      650            FL  Green Cover Springs   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "2                        25             False         False             True   \n",
       "3                         6             False         False            False   \n",
       "4                         6             False         False            False   \n",
       "5                         0              True         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "827807                    5              True         False             True   \n",
       "827808                    5              True         False             True   \n",
       "827809                    5              True         False             True   \n",
       "827810                    5              True         False             True   \n",
       "827811                    5              True         False             True   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                       Electric  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "827807                  Electric  \n",
       "827808                  Electric  \n",
       "827809                  Electric  \n",
       "827810                  Electric  \n",
       "827811                  Electric  \n",
       "\n",
       "[786123 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/FL/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/FL-day/2018-feb-day-FL.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3c2330d42e6e3d81b417b1a03edb0024793e6bbb</td>\n",
       "      <td>2018-02-22 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>805</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>FL</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a38c9e0c538e17c8d4f31a7acd6095bda8d36855</td>\n",
       "      <td>2018-02-04 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>FL</td>\n",
       "      <td>Ponte Vedra</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4227c3c3ce5b52a02f547d6a8f267836b6e7c6db</td>\n",
       "      <td>2018-02-11 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>785</td>\n",
       "      <td>735</td>\n",
       "      <td>FL</td>\n",
       "      <td>Ocoee</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9c6ff4ffc41ab995ea7860bfdf9dfff3a3f209f3</td>\n",
       "      <td>2018-02-06 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>FL</td>\n",
       "      <td>The Villages</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9d9804ca59adb6c9bcf875fafd4195177c1f42bb</td>\n",
       "      <td>2018-02-12 16:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>765</td>\n",
       "      <td>667</td>\n",
       "      <td>FL</td>\n",
       "      <td>Belleview</td>\n",
       "      <td>36</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2600953</th>\n",
       "      <td>3216be7ce3e33e00991a0c0a24aa28e14b9de11e</td>\n",
       "      <td>2018-02-28 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Cape Ciral</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2600954</th>\n",
       "      <td>a568ba12ae64af9f892bdae9c3c797b4d0bd8a3f</td>\n",
       "      <td>2018-02-17 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>bradenton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2600955</th>\n",
       "      <td>9b4541c54b4c824f2fdb592f8a1ed0ff6a7cf737</td>\n",
       "      <td>2018-02-27 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Garden</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2600956</th>\n",
       "      <td>db09654404a4bb54e8c19121626a336237a6e28e</td>\n",
       "      <td>2018-02-23 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Cape Coral</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2600957</th>\n",
       "      <td>487b18435a897f088bc6017ed35af5685eddf15d</td>\n",
       "      <td>2018-02-27 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2486382 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        3c2330d42e6e3d81b417b1a03edb0024793e6bbb  2018-02-22 18:40:00 UTC   \n",
       "1        a38c9e0c538e17c8d4f31a7acd6095bda8d36855  2018-02-04 16:50:00 UTC   \n",
       "2        4227c3c3ce5b52a02f547d6a8f267836b6e7c6db  2018-02-11 11:30:00 UTC   \n",
       "3        9c6ff4ffc41ab995ea7860bfdf9dfff3a3f209f3  2018-02-06 18:40:00 UTC   \n",
       "4        9d9804ca59adb6c9bcf875fafd4195177c1f42bb  2018-02-12 16:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2600953  3216be7ce3e33e00991a0c0a24aa28e14b9de11e  2018-02-28 18:45:00 UTC   \n",
       "2600954  a568ba12ae64af9f892bdae9c3c797b4d0bd8a3f  2018-02-17 12:10:00 UTC   \n",
       "2600955  9b4541c54b4c824f2fdb592f8a1ed0ff6a7cf737  2018-02-27 19:05:00 UTC   \n",
       "2600956  db09654404a4bb54e8c19121626a336237a6e28e  2018-02-23 15:20:00 UTC   \n",
       "2600957  487b18435a897f088bc6017ed35af5685eddf15d  2018-02-27 15:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               805                      800   \n",
       "1           auto          hold               705                      705   \n",
       "2           auto          auto               759                      785   \n",
       "3           cool          hold               740                      735   \n",
       "4           auto          hold               751                      765   \n",
       "...          ...           ...               ...                      ...   \n",
       "2600953     cool          hold               766                      760   \n",
       "2600954     cool          hold               740                      760   \n",
       "2600955     cool          hold               772                      760   \n",
       "2600956     cool          hold               762                      760   \n",
       "2600957     cool          hold               758                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            800            FL      Homestead   \n",
       "1                            655            FL    Ponte Vedra   \n",
       "2                            735            FL          Ocoee   \n",
       "3                            735            FL   The Villages   \n",
       "4                            667            FL      Belleview   \n",
       "...                          ...           ...            ...   \n",
       "2600953                      760            FL     Cape Ciral   \n",
       "2600954                      760            FL      bradenton   \n",
       "2600955                      760            FL  Winter Garden   \n",
       "2600956                      760            FL     Cape Coral   \n",
       "2600957                      760            FL         Oviedo   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False            False   \n",
       "1                          0             False         False             True   \n",
       "2                         37             False         False            False   \n",
       "3                         15             False         False            False   \n",
       "4                         36              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2600953                    8             False         False            False   \n",
       "2600954                    5             False         False             True   \n",
       "2600955                    5              True         False             True   \n",
       "2600956                    0              True         False            False   \n",
       "2600957                    8             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "2600953                       Gas  \n",
       "2600954                  Electric  \n",
       "2600955                  Electric  \n",
       "2600956                       Gas  \n",
       "2600957                  Electric  \n",
       "\n",
       "[2486382 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/FL/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/FL-day/2019-feb-day-FL.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49304c734512b21c6ad58cd7c16fe605020d5824</td>\n",
       "      <td>2019-02-20 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>759</td>\n",
       "      <td>779</td>\n",
       "      <td>779</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ddfcfffed33adf93a7d436ceb2e61ae7e8d8fda7</td>\n",
       "      <td>2019-02-18 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>722</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9803938f15ad5bfc82916a352ac6cc1e8d600249</td>\n",
       "      <td>2019-02-06 15:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>690</td>\n",
       "      <td>725</td>\n",
       "      <td>669</td>\n",
       "      <td>FL</td>\n",
       "      <td>Longwood</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1e3f36227b6d194d253f7093024dbc0dd70549df</td>\n",
       "      <td>2019-02-28 10:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lake Worth</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>cad60670d975832582759010db9533090231e9ef</td>\n",
       "      <td>2019-02-02 12:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>723</td>\n",
       "      <td>723</td>\n",
       "      <td>FL</td>\n",
       "      <td>Clermont</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907088</th>\n",
       "      <td>3216be7ce3e33e00991a0c0a24aa28e14b9de11e</td>\n",
       "      <td>2019-02-25 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Cape Ciral</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907089</th>\n",
       "      <td>b0c1b3e03e56da92f7385c04f1e316f0e05b55b5</td>\n",
       "      <td>2019-02-25 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lakeland</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907090</th>\n",
       "      <td>ab1bf48ba470aeef94f81b11565dc886061a2871</td>\n",
       "      <td>2019-02-01 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Margate</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907091</th>\n",
       "      <td>74fb118d6edaedab3da1f29e297ded25ec319791</td>\n",
       "      <td>2019-02-23 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pompano Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2907092</th>\n",
       "      <td>fe737c35ae45900e88fe927d4192abc6d40fbff5</td>\n",
       "      <td>2019-02-21 10:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2779339 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        49304c734512b21c6ad58cd7c16fe605020d5824  2019-02-20 13:40:00 UTC   \n",
       "2        ddfcfffed33adf93a7d436ceb2e61ae7e8d8fda7  2019-02-18 16:30:00 UTC   \n",
       "3        9803938f15ad5bfc82916a352ac6cc1e8d600249  2019-02-06 15:05:00 UTC   \n",
       "4        1e3f36227b6d194d253f7093024dbc0dd70549df  2019-02-28 10:40:00 UTC   \n",
       "7        cad60670d975832582759010db9533090231e9ef  2019-02-02 12:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2907088  3216be7ce3e33e00991a0c0a24aa28e14b9de11e  2019-02-25 18:45:00 UTC   \n",
       "2907089  b0c1b3e03e56da92f7385c04f1e316f0e05b55b5  2019-02-25 13:05:00 UTC   \n",
       "2907090  ab1bf48ba470aeef94f81b11565dc886061a2871  2019-02-01 15:25:00 UTC   \n",
       "2907091  74fb118d6edaedab3da1f29e297ded25ec319791  2019-02-23 17:10:00 UTC   \n",
       "2907092  fe737c35ae45900e88fe927d4192abc6d40fbff5  2019-02-21 10:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               759                      779   \n",
       "2           cool          hold               722                      722   \n",
       "3           auto          hold               690                      725   \n",
       "4           auto          auto               733                      755   \n",
       "7           heat          hold               728                      723   \n",
       "...          ...           ...               ...                      ...   \n",
       "2907088     cool          hold               759                      760   \n",
       "2907089     cool          auto               753                      760   \n",
       "2907090     cool          auto               739                      760   \n",
       "2907091     cool          hold               755                      760   \n",
       "2907092     cool          auto               754                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            779            FL        Orlando   \n",
       "2                            722            FL         Naples   \n",
       "3                            669            FL       Longwood   \n",
       "4                            705            FL     Lake Worth   \n",
       "7                            723            FL       Clermont   \n",
       "...                          ...           ...            ...   \n",
       "2907088                      760            FL     Cape Ciral   \n",
       "2907089                      760            FL       Lakeland   \n",
       "2907090                      760            FL        Margate   \n",
       "2907091                      760            FL  Pompano Beach   \n",
       "2907092                      760            FL      Kissimmee   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10              True         False            False   \n",
       "2                          9              True         False            False   \n",
       "3                         40             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "7                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2907088                    8             False         False            False   \n",
       "2907089                    0             False         False             True   \n",
       "2907090                   50             False         False            False   \n",
       "2907091                    0             False         False            False   \n",
       "2907092                   25             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "7                        Electric  \n",
       "...                           ...  \n",
       "2907088                       Gas  \n",
       "2907089                  Electric  \n",
       "2907090                       Gas  \n",
       "2907091                       Gas  \n",
       "2907092                  Electric  \n",
       "\n",
       "[2779339 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/FL/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/FL-day/2020-feb-day-FL.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19078efc342b5dd095f36c5b2958a3558a39613c</td>\n",
       "      <td>2020-02-19 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Haven</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>44fa42a91646206f8047908b2359763cbb2bebdd</td>\n",
       "      <td>2020-02-01 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>721</td>\n",
       "      <td>719</td>\n",
       "      <td>659</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>805b979c88c6beb513fb8be8963da99c2cb05798</td>\n",
       "      <td>2020-02-11 13:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>702</td>\n",
       "      <td>720</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cover Springs</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c0472acb00956145e2b27225459610c22c636e98</td>\n",
       "      <td>2020-02-09 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>721</td>\n",
       "      <td>740</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tierra Verde</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49c4c11b8b27e2b888a6a92ae01f4741d4ac69e3</td>\n",
       "      <td>2020-02-15 12:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami Beach</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998898</th>\n",
       "      <td>63a9630ed3ea7381f7a6f9aee56c6c5d25c39d8d</td>\n",
       "      <td>2020-02-23 18:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>678</td>\n",
       "      <td>735</td>\n",
       "      <td>685</td>\n",
       "      <td>FL</td>\n",
       "      <td>Holmes Beach</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998899</th>\n",
       "      <td>75774d2daf04754cdd5ebdcdc74ab62c7dc883c7</td>\n",
       "      <td>2020-02-09 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>731</td>\n",
       "      <td>750</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lehigh Acres</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998900</th>\n",
       "      <td>f79bdd3708b2e9b9adbf35a895b6b6929351aa31</td>\n",
       "      <td>2020-02-22 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>743</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998901</th>\n",
       "      <td>630ea436ad4ed1e37802d2cc3d2f92c0bc9777f8</td>\n",
       "      <td>2020-02-02 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pompano Beach</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998902</th>\n",
       "      <td>124766677c0002fd5cf886ce535d893c180f0d87</td>\n",
       "      <td>2020-02-23 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>692</td>\n",
       "      <td>740</td>\n",
       "      <td>690</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Petersburg</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3853429 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        19078efc342b5dd095f36c5b2958a3558a39613c  2020-02-19 16:10:00 UTC   \n",
       "1        44fa42a91646206f8047908b2359763cbb2bebdd  2020-02-01 18:00:00 UTC   \n",
       "2        805b979c88c6beb513fb8be8963da99c2cb05798  2020-02-11 13:20:00 UTC   \n",
       "3        c0472acb00956145e2b27225459610c22c636e98  2020-02-09 12:15:00 UTC   \n",
       "4        49c4c11b8b27e2b888a6a92ae01f4741d4ac69e3  2020-02-15 12:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "3998898  63a9630ed3ea7381f7a6f9aee56c6c5d25c39d8d  2020-02-23 18:10:00 UTC   \n",
       "3998899  75774d2daf04754cdd5ebdcdc74ab62c7dc883c7  2020-02-09 15:25:00 UTC   \n",
       "3998900  f79bdd3708b2e9b9adbf35a895b6b6929351aa31  2020-02-22 14:15:00 UTC   \n",
       "3998901  630ea436ad4ed1e37802d2cc3d2f92c0bc9777f8  2020-02-02 15:30:00 UTC   \n",
       "3998902  124766677c0002fd5cf886ce535d893c180f0d87  2020-02-23 15:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               703                      700   \n",
       "1           auto          hold               721                      719   \n",
       "2           auto          auto               702                      720   \n",
       "3           auto          auto               721                      740   \n",
       "4           cool          auto               703                      700   \n",
       "...          ...           ...               ...                      ...   \n",
       "3998898     auto          auto               678                      735   \n",
       "3998899     cool          hold               731                      750   \n",
       "3998900     heat          auto               743                      750   \n",
       "3998901     cool          hold               683                      680   \n",
       "3998902     heat          auto               692                      740   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            750            FL         Winter Haven   \n",
       "1                            659            FL            Kissimmee   \n",
       "2                            700            FL  Green Cover Springs   \n",
       "3                            680            FL         Tierra Verde   \n",
       "4                            700            FL          Miami Beach   \n",
       "...                          ...           ...                  ...   \n",
       "3998898                      685            FL         Holmes Beach   \n",
       "3998899                      730            FL         Lehigh Acres   \n",
       "3998900                      750            FL                Tampa   \n",
       "3998901                      680            FL        Pompano Beach   \n",
       "3998902                      690            FL     Saint Petersburg   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False             True   \n",
       "1                          0             False         False            False   \n",
       "2                          5              True         False             True   \n",
       "3                         30              True         False            False   \n",
       "4                          9             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "3998898                   30             False         False            False   \n",
       "3998899                   19              True         False            False   \n",
       "3998900                   35              True         False             True   \n",
       "3998901                   40             False         False            False   \n",
       "3998902                   50             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "3998898                       Gas  \n",
       "3998899                       Gas  \n",
       "3998900                  Electric  \n",
       "3998901                       Gas  \n",
       "3998902                       Gas  \n",
       "\n",
       "[3853429 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/FL/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/FL-day/2021-feb-day-FL.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2e21444d04b9963cfc5948da4e91962c3ae122b8</td>\n",
       "      <td>2021-02-28 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>766</td>\n",
       "      <td>766</td>\n",
       "      <td>FL</td>\n",
       "      <td>Panama City</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bc569bfaf3396e0ecca46e88a333f31a5f984d24</td>\n",
       "      <td>2021-02-11 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>762</td>\n",
       "      <td>762</td>\n",
       "      <td>FL</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78b50b5a677d87e7551186383ecf31fd1e014b45</td>\n",
       "      <td>2021-02-18 15:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>808</td>\n",
       "      <td>708</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tarpon Springs</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>87a9a530caa8878f9628838d182ae0afbced2f90</td>\n",
       "      <td>2021-02-27 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>743</td>\n",
       "      <td>608</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Springs</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>179bf8ccb2198aee1c0053d4a79ad13116630d34</td>\n",
       "      <td>2021-02-07 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>602</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2331063</th>\n",
       "      <td>d30b1602fbdca6b250f1ff278fc389f84ae44493</td>\n",
       "      <td>2021-02-02 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Windermere</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2331064</th>\n",
       "      <td>a9e0c1277d7c419c9c53a177622342ee019c3418</td>\n",
       "      <td>2021-02-15 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>770</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Cloud</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2331065</th>\n",
       "      <td>38a68c85ecd6047a0c8823b7787e7d68e320a234</td>\n",
       "      <td>2021-02-25 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Boca Raton</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2331066</th>\n",
       "      <td>13ab23a3b0b55d9169ad1d4441a7f4f1585d1fb3</td>\n",
       "      <td>2021-02-10 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pompano Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2331067</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>2021-02-22 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2202938 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        2e21444d04b9963cfc5948da4e91962c3ae122b8  2021-02-28 19:25:00 UTC   \n",
       "2        bc569bfaf3396e0ecca46e88a333f31a5f984d24  2021-02-11 16:20:00 UTC   \n",
       "3        78b50b5a677d87e7551186383ecf31fd1e014b45  2021-02-18 15:15:00 UTC   \n",
       "4        87a9a530caa8878f9628838d182ae0afbced2f90  2021-02-27 17:10:00 UTC   \n",
       "5        179bf8ccb2198aee1c0053d4a79ad13116630d34  2021-02-07 17:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2331063  d30b1602fbdca6b250f1ff278fc389f84ae44493  2021-02-02 13:45:00 UTC   \n",
       "2331064  a9e0c1277d7c419c9c53a177622342ee019c3418  2021-02-15 17:20:00 UTC   \n",
       "2331065  38a68c85ecd6047a0c8823b7787e7d68e320a234  2021-02-25 17:30:00 UTC   \n",
       "2331066  13ab23a3b0b55d9169ad1d4441a7f4f1585d1fb3  2021-02-10 19:40:00 UTC   \n",
       "2331067  ffcf942aa8736eaa25440e6817dea4d9efc3c67f  2021-02-22 18:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               725                      766   \n",
       "2           cool          hold               749                      762   \n",
       "3           auto          hold               749                      808   \n",
       "4           cool          hold               755                      743   \n",
       "5           cool          hold               765                      760   \n",
       "...          ...           ...               ...                      ...   \n",
       "2331063     heat          hold               758                      760   \n",
       "2331064     cool          hold               764                      770   \n",
       "2331065     cool          hold               760                      760   \n",
       "2331066     cool          hold               764                      760   \n",
       "2331067     cool          hold               756                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            766            FL     Panama City   \n",
       "2                            762            FL      Homestead    \n",
       "3                            708            FL  Tarpon Springs   \n",
       "4                            608            FL   Coral Springs   \n",
       "5                            602            FL          Naples   \n",
       "...                          ...           ...             ...   \n",
       "2331063                      760            FL      Windermere   \n",
       "2331064                      760            FL     Saint Cloud   \n",
       "2331065                      760            FL      Boca Raton   \n",
       "2331066                      760            FL   Pompano Beach   \n",
       "2331067                      760            FL    Oakland Park   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         15             False         False             True   \n",
       "2                         10             False         False            False   \n",
       "3                         10             False         False             True   \n",
       "4                         35             False         False            False   \n",
       "5                          0              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2331063                   10             False         False            False   \n",
       "2331064                    5             False         False             True   \n",
       "2331065                   40             False         False            False   \n",
       "2331066                    0             False         False            False   \n",
       "2331067                   30             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "2331063                       Gas  \n",
       "2331064                  Electric  \n",
       "2331065                       Gas  \n",
       "2331066                       Gas  \n",
       "2331067                       Gas  \n",
       "\n",
       "[2202938 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/FL/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>722.084507</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>767.633484</td>\n",
       "      <td>799.990950</td>\n",
       "      <td>760.009050</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>749.417382</td>\n",
       "      <td>774.985515</td>\n",
       "      <td>774.975858</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>01047c4cced05cfd81b6a7c9c263239317203975</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>747.507784</td>\n",
       "      <td>761.637126</td>\n",
       "      <td>694.265868</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01047c4cced05cfd81b6a7c9c263239317203975</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>751.039216</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>709.054054</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>756.978571</td>\n",
       "      <td>759.995238</td>\n",
       "      <td>759.980952</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>723.718160</td>\n",
       "      <td>725.439858</td>\n",
       "      <td>724.521226</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>729.333333</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>730.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3830</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>763.550000</td>\n",
       "      <td>759.500000</td>\n",
       "      <td>759.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22630 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   feb  2017     auto   \n",
       "1     0024f0f900a4ff6923e95a2a20ee94c699ecf167   feb  2017     cool   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   feb  2017     cool   \n",
       "3     01047c4cced05cfd81b6a7c9c263239317203975   feb  2017     cool   \n",
       "4     01047c4cced05cfd81b6a7c9c263239317203975   feb  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "3826  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   feb  2021     auto   \n",
       "3827  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   feb  2021     cool   \n",
       "3828  fffcd8322781a8483039a10c259725e9da9cfdec   feb  2021     cool   \n",
       "3829  ffff46e404a9831c28d8be1a3e427a4377690c6b   feb  2021     auto   \n",
       "3830  ffff46e404a9831c28d8be1a3e427a4377690c6b   feb  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            722.084507   \n",
       "1             auto         Miami            767.633484   \n",
       "2             hold         Miami            749.417382   \n",
       "3             auto        Oviedo            747.507784   \n",
       "4             hold        Oviedo            751.039216   \n",
       "...            ...           ...                   ...   \n",
       "3826          hold  Oakland Park            709.054054   \n",
       "3827          hold  Oakland Park            756.978571   \n",
       "3828          hold   North Miami            723.718160   \n",
       "3829          hold       Parrish            729.333333   \n",
       "3830          hold       Parrish            763.550000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      760.000000                   670.000000   \n",
       "1                      799.990950                   760.009050   \n",
       "2                      774.985515                   774.975858   \n",
       "3                      761.637126                   694.265868   \n",
       "4                      760.000000                   760.000000   \n",
       "...                           ...                          ...   \n",
       "3826                   760.000000                   680.000000   \n",
       "3827                   759.995238                   759.980952   \n",
       "3828                   725.439858                   724.521226   \n",
       "3829                   780.000000                   730.000000   \n",
       "3830                   759.500000                   759.500000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     0.0             False         False             True  \n",
       "4                     0.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "3826                 30.0             False         False            False  \n",
       "3827                 30.0             False         False            False  \n",
       "3828                 70.0             False         False            False  \n",
       "3829                  0.0             False         False             True  \n",
       "3830                  0.0             False         False             True  \n",
       "\n",
       "[22630 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/feb/\" + file)\n",
    "    FL_feb = pd.concat([FL_feb, df])\n",
    "    \n",
    "FL_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_feb.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/FL-day/2017-jun-day-FL.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12a37145fd7cee5f696f62c41b25b77e7884801a</td>\n",
       "      <td>2017-06-15 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>780</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6fe42f2a5734babc53ae9e179fb5c61dd5aa2bce</td>\n",
       "      <td>2017-06-05 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>773</td>\n",
       "      <td>770</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Daytona Beach</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1d71d6b47aa76f9be2339f37a8a4d984bab6f050</td>\n",
       "      <td>2017-06-09 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Casselberry</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2dbdc0c5b9ee64661a516a3fb359cc277316a415</td>\n",
       "      <td>2017-06-01 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Destin</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2616da840190b037029be47d0bddbd80e2dd29e9</td>\n",
       "      <td>2017-06-27 19:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>790</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Altamonte Springs</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640739</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-06-11 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640740</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-06-06 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640741</th>\n",
       "      <td>b68773762e82ce89bfde709432c259f719afb7d0</td>\n",
       "      <td>2017-06-12 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640742</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-06-28 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>730</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1640743</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-06-23 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1576912 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        12a37145fd7cee5f696f62c41b25b77e7884801a  2017-06-15 12:25:00 UTC   \n",
       "1        6fe42f2a5734babc53ae9e179fb5c61dd5aa2bce  2017-06-05 19:35:00 UTC   \n",
       "2        1d71d6b47aa76f9be2339f37a8a4d984bab6f050  2017-06-09 16:45:00 UTC   \n",
       "3        2dbdc0c5b9ee64661a516a3fb359cc277316a415  2017-06-01 14:35:00 UTC   \n",
       "4        2616da840190b037029be47d0bddbd80e2dd29e9  2017-06-27 19:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1640739  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-06-11 18:00:00 UTC   \n",
       "1640740  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-06-06 14:00:00 UTC   \n",
       "1640741  b68773762e82ce89bfde709432c259f719afb7d0  2017-06-12 12:15:00 UTC   \n",
       "1640742  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-06-28 17:15:00 UTC   \n",
       "1640743  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-06-23 10:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               762                      780   \n",
       "1           cool          auto               773                      770   \n",
       "2           cool          hold               755                      750   \n",
       "3           cool          auto               732                      760   \n",
       "4           auto          auto               790                      760   \n",
       "...          ...           ...               ...                      ...   \n",
       "1640739     cool          hold               743                      740   \n",
       "1640740     cool          hold               744                      740   \n",
       "1640741     auto          hold               764                      760   \n",
       "1640742     cool          auto               739                      730   \n",
       "1640743     cool          hold               730                      730   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            700            FL       Coral Springs   \n",
       "1                            700            FL       Daytona Beach   \n",
       "2                            750            FL         Casselberry   \n",
       "3                            710            FL              Destin   \n",
       "4                            680            FL   Altamonte Springs   \n",
       "...                          ...           ...                 ...   \n",
       "1640739                      740            FL  Palm Beach Gardens   \n",
       "1640740                      740            FL  Palm Beach Gardens   \n",
       "1640741                      620            FL  Palm Beach Gardens   \n",
       "1640742                      740            FL  Palm Beach Gardens   \n",
       "1640743                      730            FL  Palm Beach Gardens   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                          5             False         False             True   \n",
       "2                         30             False         False             True   \n",
       "3                         20             False         False             True   \n",
       "4                         27             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1640739                    0             False         False            False   \n",
       "1640740                    0             False         False            False   \n",
       "1640741                    5              True         False            False   \n",
       "1640742                    0             False         False            False   \n",
       "1640743                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1640739                       Gas  \n",
       "1640740                       Gas  \n",
       "1640741                       Gas  \n",
       "1640742                       Gas  \n",
       "1640743                       Gas  \n",
       "\n",
       "[1576912 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/FL/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/FL-day/2018-jun-day-FL.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d3fda431d007897e050b6423e569cf772d1eb6aa</td>\n",
       "      <td>2018-06-16 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>746</td>\n",
       "      <td>745</td>\n",
       "      <td>635</td>\n",
       "      <td>FL</td>\n",
       "      <td>Gainesville</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b4cfc4cc0bc94157625d11f8644ce9dd6e4fb311</td>\n",
       "      <td>2018-06-01 17:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>777</td>\n",
       "      <td>770</td>\n",
       "      <td>702</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ea12225706ab26195d1b58fb72e36724a6215482</td>\n",
       "      <td>2018-06-21 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Springs</td>\n",
       "      <td>29</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c42808e540740a1918cc4a82c53ec5b93f3af64c</td>\n",
       "      <td>2018-06-25 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>776</td>\n",
       "      <td>772</td>\n",
       "      <td>722</td>\n",
       "      <td>FL</td>\n",
       "      <td>Middleburg</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2ae3555512164c3351067061f9a9525ade404bee</td>\n",
       "      <td>2018-06-16 12:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>FL</td>\n",
       "      <td>Bal Harbour</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723115</th>\n",
       "      <td>b587ccf0e72a372eb7e08a4abc1ea0d9871f52c4</td>\n",
       "      <td>2018-06-19 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Doral</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723116</th>\n",
       "      <td>dbcf059b3900ee79bc9e7ec66067b3c5f417d173</td>\n",
       "      <td>2018-06-25 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>771</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>38</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723117</th>\n",
       "      <td>a59f6ea94a6f1bde3105f36b849b7219b22bdce9</td>\n",
       "      <td>2018-06-30 10:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Clermont</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723118</th>\n",
       "      <td>29f4e7036c69a1829157ac29c4fd3f7c234ed3c3</td>\n",
       "      <td>2018-06-25 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>North Palm Beach</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723119</th>\n",
       "      <td>9a20b100e8d68607ae97184e7136ea905e6891d1</td>\n",
       "      <td>2018-06-15 19:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3570506 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        d3fda431d007897e050b6423e569cf772d1eb6aa  2018-06-16 17:35:00 UTC   \n",
       "1        b4cfc4cc0bc94157625d11f8644ce9dd6e4fb311  2018-06-01 17:15:00 UTC   \n",
       "2        ea12225706ab26195d1b58fb72e36724a6215482  2018-06-21 15:00:00 UTC   \n",
       "3        c42808e540740a1918cc4a82c53ec5b93f3af64c  2018-06-25 17:30:00 UTC   \n",
       "4        2ae3555512164c3351067061f9a9525ade404bee  2018-06-16 12:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "3723115  b587ccf0e72a372eb7e08a4abc1ea0d9871f52c4  2018-06-19 13:30:00 UTC   \n",
       "3723116  dbcf059b3900ee79bc9e7ec66067b3c5f417d173  2018-06-25 17:15:00 UTC   \n",
       "3723117  a59f6ea94a6f1bde3105f36b849b7219b22bdce9  2018-06-30 10:45:00 UTC   \n",
       "3723118  29f4e7036c69a1829157ac29c4fd3f7c234ed3c3  2018-06-25 18:25:00 UTC   \n",
       "3723119  9a20b100e8d68607ae97184e7136ea905e6891d1  2018-06-15 19:20:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               746                      745   \n",
       "1           auto          hold               777                      770   \n",
       "2           cool          hold               717                      705   \n",
       "3           auto          auto               776                      772   \n",
       "4           cool          hold               694                      689   \n",
       "...          ...           ...               ...                      ...   \n",
       "3723115     cool          auto               741                      760   \n",
       "3723116     cool          auto               771                      760   \n",
       "3723117     cool          hold               760                      760   \n",
       "3723118     cool          hold               758                      760   \n",
       "3723119     cool          hold               759                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            635            FL       Gainesville   \n",
       "1                            702            FL             Miami   \n",
       "2                            705            FL     Coral Springs   \n",
       "3                            722            FL        Middleburg   \n",
       "4                            689            FL       Bal Harbour   \n",
       "...                          ...           ...               ...   \n",
       "3723115                      760            FL             Doral   \n",
       "3723116                      760            FL             Miami   \n",
       "3723117                      760            FL          Clermont   \n",
       "3723118                      760            FL  North Palm Beach   \n",
       "3723119                      760            FL      Jacksonville   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30             False         False            False   \n",
       "1                         57             False         False            False   \n",
       "2                         29              True         False            False   \n",
       "3                          5              True         False             True   \n",
       "4                         50             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "3723115                    5              True         False            False   \n",
       "3723116                   38             False         False            False   \n",
       "3723117                   20             False         False            False   \n",
       "3723118                   67             False         False            False   \n",
       "3723119                    0              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "3723115                       Gas  \n",
       "3723116                       Gas  \n",
       "3723117                       Gas  \n",
       "3723118                       Gas  \n",
       "3723119                  Electric  \n",
       "\n",
       "[3570506 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/FL/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/FL-day/2019-jun-day-FL.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b2f11a2b83b3051b1bd11e541e2952dbd58acfe9</td>\n",
       "      <td>2019-06-22 10:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>720</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d6c2e44b80e82e437e07bc818b818327a926c101</td>\n",
       "      <td>2019-06-10 10:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>779</td>\n",
       "      <td>780</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4c2630ecafeab7d1b1c1ba43ea74b6824b73710a</td>\n",
       "      <td>2019-06-26 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>778</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Riviera Beach</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3afd00dfb040a8ee659232ba6cffa6fb62e761c2</td>\n",
       "      <td>2019-06-08 13:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>FL</td>\n",
       "      <td>Eustis</td>\n",
       "      <td>14</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>d1e41a9099613d8dfb3f4b2f09d9391594040ed8</td>\n",
       "      <td>2019-06-26 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>732</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Gainesville</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5338418</th>\n",
       "      <td>8adbf7ff551bba4f616e7317bb628efea3ada6ae</td>\n",
       "      <td>2019-06-20 15:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5338419</th>\n",
       "      <td>3585c81d9c8e352062dd4b76e4ab5bc153f60a0c</td>\n",
       "      <td>2019-06-17 11:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>686</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Davie</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5338420</th>\n",
       "      <td>9a2a5817132918aa8b4f7d83b5daf5e532b28603</td>\n",
       "      <td>2019-06-25 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pinecrest</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5338421</th>\n",
       "      <td>36b5f816c2bd952f875b0691d0d94aa7b2376b1f</td>\n",
       "      <td>2019-06-03 10:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>731</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Safety Harbor</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5338422</th>\n",
       "      <td>45c906891e209cfa791d4d11f4b7795c6112c483</td>\n",
       "      <td>2019-06-02 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>847</td>\n",
       "      <td>840</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Apollo Beach</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5086284 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        b2f11a2b83b3051b1bd11e541e2952dbd58acfe9  2019-06-22 10:40:00 UTC   \n",
       "1        d6c2e44b80e82e437e07bc818b818327a926c101  2019-06-10 10:55:00 UTC   \n",
       "3        4c2630ecafeab7d1b1c1ba43ea74b6824b73710a  2019-06-26 17:55:00 UTC   \n",
       "4        3afd00dfb040a8ee659232ba6cffa6fb62e761c2  2019-06-08 13:40:00 UTC   \n",
       "5        d1e41a9099613d8dfb3f4b2f09d9391594040ed8  2019-06-26 18:45:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "5338418  8adbf7ff551bba4f616e7317bb628efea3ada6ae  2019-06-20 15:00:00 UTC   \n",
       "5338419  3585c81d9c8e352062dd4b76e4ab5bc153f60a0c  2019-06-17 11:30:00 UTC   \n",
       "5338420  9a2a5817132918aa8b4f7d83b5daf5e532b28603  2019-06-25 13:25:00 UTC   \n",
       "5338421  36b5f816c2bd952f875b0691d0d94aa7b2376b1f  2019-06-03 10:45:00 UTC   \n",
       "5338422  45c906891e209cfa791d4d11f4b7795c6112c483  2019-06-02 13:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               724                      720   \n",
       "1           auto          auto               779                      780   \n",
       "3           cool          auto               778                      780   \n",
       "4           auto          hold               762                      760   \n",
       "5           cool          auto               732                      730   \n",
       "...          ...           ...               ...                      ...   \n",
       "5338418     cool          hold               775                      770   \n",
       "5338419     cool          auto               686                      710   \n",
       "5338420     cool          hold               730                      730   \n",
       "5338421     auto          auto               731                      730   \n",
       "5338422     cool          auto               847                      840   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            650            FL          Tampa   \n",
       "1                            710            FL      Homestead   \n",
       "3                            780            FL  Riviera Beach   \n",
       "4                            620            FL         Eustis   \n",
       "5                            730            FL    Gainesville   \n",
       "...                          ...           ...            ...   \n",
       "5338418                      770            FL          Tampa   \n",
       "5338419                      710            FL          Davie   \n",
       "5338420                      730            FL      Pinecrest   \n",
       "5338421                      680            FL  Safety Harbor   \n",
       "5338422                      790            FL   Apollo Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5             False         False             True   \n",
       "1                          5             False         False            False   \n",
       "3                          9              True         False            False   \n",
       "4                         14              True         False             True   \n",
       "5                          9              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "5338418                   15              True         False             True   \n",
       "5338419                   70             False         False            False   \n",
       "5338420                    0             False         False            False   \n",
       "5338421                   30             False         False            False   \n",
       "5338422                    5             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "5338418                  Electric  \n",
       "5338419                       Gas  \n",
       "5338420                       Gas  \n",
       "5338421                       Gas  \n",
       "5338422                  Electric  \n",
       "\n",
       "[5086284 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/FL/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/FL-day/2020-jun-day-FL.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>90784eeb53e8b33dbc4b94a9cf2cca66be14b0c7</td>\n",
       "      <td>2020-06-28 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>727</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Brandon</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>03849d3f5ead7bde83ca3a9452958a62e7b2ae7b</td>\n",
       "      <td>2020-06-02 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>744</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lakeland</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dc285fea74057193e34044a37250b5c27be6e2dd</td>\n",
       "      <td>2020-06-25 13:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>740</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Safety Harbor</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49bc8e207e437c504d2c3a3603bdbf0878016e17</td>\n",
       "      <td>2020-06-26 14:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>703</td>\n",
       "      <td>675</td>\n",
       "      <td>645</td>\n",
       "      <td>FL</td>\n",
       "      <td>Destin</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7e6178e51ea702589f93017496790afe2dfc93b0</td>\n",
       "      <td>2020-06-03 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>780</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4828512</th>\n",
       "      <td>7afb3cb57a50d2aec2e614fa2f58dc0967534384</td>\n",
       "      <td>2020-06-16 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Augustine</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4828513</th>\n",
       "      <td>31fcf11dc1c1977e1e92f996eb47f5b7c4925437</td>\n",
       "      <td>2020-06-02 19:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>660</td>\n",
       "      <td>FL</td>\n",
       "      <td>Windermere</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4828514</th>\n",
       "      <td>5e6b54a039acb6f90d2fcaaf2461d3dfc5e29dfe</td>\n",
       "      <td>2020-06-07 10:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>770</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Harbor</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4828515</th>\n",
       "      <td>f234070440bf10761f0aa33313d1456ffe7eb73c</td>\n",
       "      <td>2020-06-30 11:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>DeBary</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4828516</th>\n",
       "      <td>8a188da39be9d3c57e82aee48ec5d5b3f6384c72</td>\n",
       "      <td>2020-06-08 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4632580 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        90784eeb53e8b33dbc4b94a9cf2cca66be14b0c7  2020-06-28 10:50:00 UTC   \n",
       "2        03849d3f5ead7bde83ca3a9452958a62e7b2ae7b  2020-06-02 17:25:00 UTC   \n",
       "3        dc285fea74057193e34044a37250b5c27be6e2dd  2020-06-25 13:35:00 UTC   \n",
       "4        49bc8e207e437c504d2c3a3603bdbf0878016e17  2020-06-26 14:00:00 UTC   \n",
       "5        7e6178e51ea702589f93017496790afe2dfc93b0  2020-06-03 19:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4828512  7afb3cb57a50d2aec2e614fa2f58dc0967534384  2020-06-16 19:25:00 UTC   \n",
       "4828513  31fcf11dc1c1977e1e92f996eb47f5b7c4925437  2020-06-02 19:45:00 UTC   \n",
       "4828514  5e6b54a039acb6f90d2fcaaf2461d3dfc5e29dfe  2020-06-07 10:15:00 UTC   \n",
       "4828515  f234070440bf10761f0aa33313d1456ffe7eb73c  2020-06-30 11:20:00 UTC   \n",
       "4828516  8a188da39be9d3c57e82aee48ec5d5b3f6384c72  2020-06-08 12:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           cool          auto               727                      750   \n",
       "2           cool          hold               744                      740   \n",
       "3           auto          auto               739                      740   \n",
       "4           cool          auto               703                      675   \n",
       "5           auto          hold               778                      780   \n",
       "...          ...           ...               ...                      ...   \n",
       "4828512     cool          hold               718                      720   \n",
       "4828513     auto          hold               740                      740   \n",
       "4828514     auto          hold               765                      770   \n",
       "4828515     auto          hold               780                      780   \n",
       "4828516     cool          hold               754                      750   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "1                            750            FL          Brandon   \n",
       "2                            740            FL         Lakeland   \n",
       "3                            670            FL    Safety Harbor   \n",
       "4                            645            FL           Destin   \n",
       "5                            680            FL        Kissimmee   \n",
       "...                          ...           ...              ...   \n",
       "4828512                      720            FL  Saint Augustine   \n",
       "4828513                      660            FL       Windermere   \n",
       "4828514                      680            FL      Palm Harbor   \n",
       "4828515                      710            FL           DeBary   \n",
       "4828516                      750            FL     Oakland Park   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         55             False         False            False   \n",
       "2                         39             False         False             True   \n",
       "3                         39             False         False             True   \n",
       "4                          0             False         False             True   \n",
       "5                          5             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "4828512                    9              True         False             True   \n",
       "4828513                    0              True         False             True   \n",
       "4828514                   20             False         False             True   \n",
       "4828515                   10             False         False             True   \n",
       "4828516                   55             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "5                        Electric  \n",
       "...                           ...  \n",
       "4828512                  Electric  \n",
       "4828513                  Electric  \n",
       "4828514                  Electric  \n",
       "4828515                  Electric  \n",
       "4828516                       Gas  \n",
       "\n",
       "[4632580 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/FL/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/FL-day/2021-jun-day-FL.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>034922a0328fa08cd6e02d903796ae49a76310d3</td>\n",
       "      <td>2021-06-23 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>739</td>\n",
       "      <td>739</td>\n",
       "      <td>FL</td>\n",
       "      <td>Fort Lauderdale</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ef503d0d2eea377b740af158f5dbf2580b771fc9</td>\n",
       "      <td>2021-06-22 16:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>749</td>\n",
       "      <td>649</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>179bf8ccb2198aee1c0053d4a79ad13116630d34</td>\n",
       "      <td>2021-06-27 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>771</td>\n",
       "      <td>770</td>\n",
       "      <td>602</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2d6b9c2877603139fc936a1dca5018f61e90dc4d</td>\n",
       "      <td>2021-06-12 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>770</td>\n",
       "      <td>768</td>\n",
       "      <td>FL</td>\n",
       "      <td>Fort Myers</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0224817a4b93ae69c26f2532615f7d7ab28201a9</td>\n",
       "      <td>2021-06-10 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>FL</td>\n",
       "      <td>Sunrise</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880560</th>\n",
       "      <td>1173246850f4de7086b3d1ce621458f7846f368b</td>\n",
       "      <td>2021-06-13 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Gables</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880561</th>\n",
       "      <td>7da30854b2387f9ae355e3d02197145074a5b5c9</td>\n",
       "      <td>2021-06-09 08:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Melbourne</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880562</th>\n",
       "      <td>0f06f312e027e0b6d3b572ef5dd2aa01c9dd0879</td>\n",
       "      <td>2021-06-04 19:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880563</th>\n",
       "      <td>de8254a96f114f8f1dbaa7c92d56196276e40b86</td>\n",
       "      <td>2021-06-30 14:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>west palm beach</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2880564</th>\n",
       "      <td>9c95edce20eb9b90e1a68b5a405636963691c7e6</td>\n",
       "      <td>2021-06-16 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miramar</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2755167 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        034922a0328fa08cd6e02d903796ae49a76310d3  2021-06-23 17:50:00 UTC   \n",
       "1        ef503d0d2eea377b740af158f5dbf2580b771fc9  2021-06-22 16:45:00 UTC   \n",
       "2        179bf8ccb2198aee1c0053d4a79ad13116630d34  2021-06-27 15:25:00 UTC   \n",
       "3        2d6b9c2877603139fc936a1dca5018f61e90dc4d  2021-06-12 16:05:00 UTC   \n",
       "4        0224817a4b93ae69c26f2532615f7d7ab28201a9  2021-06-10 16:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2880560  1173246850f4de7086b3d1ce621458f7846f368b  2021-06-13 18:45:00 UTC   \n",
       "2880561  7da30854b2387f9ae355e3d02197145074a5b5c9  2021-06-09 08:35:00 UTC   \n",
       "2880562  0f06f312e027e0b6d3b572ef5dd2aa01c9dd0879  2021-06-04 19:10:00 UTC   \n",
       "2880563  de8254a96f114f8f1dbaa7c92d56196276e40b86  2021-06-30 14:45:00 UTC   \n",
       "2880564  9c95edce20eb9b90e1a68b5a405636963691c7e6  2021-06-16 19:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               751                      739   \n",
       "1           auto          hold               752                      749   \n",
       "2           cool          hold               771                      770   \n",
       "3           cool          hold               772                      770   \n",
       "4           cool          hold               718                      712   \n",
       "...          ...           ...               ...                      ...   \n",
       "2880560     cool          hold               760                      760   \n",
       "2880561     cool          hold               761                      760   \n",
       "2880562     cool          hold               761                      760   \n",
       "2880563     cool          hold               754                      760   \n",
       "2880564     cool          hold               763                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            739            FL  Fort Lauderdale   \n",
       "1                            649            FL       Wellington   \n",
       "2                            602            FL           Naples   \n",
       "3                            768            FL       Fort Myers   \n",
       "4                            712            FL          Sunrise   \n",
       "...                          ...           ...              ...   \n",
       "2880560                      760            FL     Coral Gables   \n",
       "2880561                      760            FL        Melbourne   \n",
       "2880562                      760            FL           Naples   \n",
       "2880563                      760            FL  west palm beach   \n",
       "2880564                      760            FL          Miramar   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         40             False         False            False   \n",
       "1                         10             False         False            False   \n",
       "2                          0              True         False            False   \n",
       "3                         25              True         False            False   \n",
       "4                         40             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2880560                   60             False         False            False   \n",
       "2880561                   67             False         False            False   \n",
       "2880562                    0             False         False            False   \n",
       "2880563                   60              True         False            False   \n",
       "2880564                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2880560                       Gas  \n",
       "2880561                       Gas  \n",
       "2880562                       Gas  \n",
       "2880563                       Gas  \n",
       "2880564                       Gas  \n",
       "\n",
       "[2755167 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/FL/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>775.492308</td>\n",
       "      <td>773.423077</td>\n",
       "      <td>667.163462</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>799.983353</td>\n",
       "      <td>804.537455</td>\n",
       "      <td>788.787158</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>796.289140</td>\n",
       "      <td>806.028507</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>753.254545</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>008b53c11b3b4c56fac3ef938ccd9a34d7755d30</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Davie</td>\n",
       "      <td>739.529412</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>690.000000</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2875</th>\n",
       "      <td>ff5c81c4eacfb17ff63cecf31c131332abc79ddc</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Myers</td>\n",
       "      <td>743.019139</td>\n",
       "      <td>739.555024</td>\n",
       "      <td>652.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2876</th>\n",
       "      <td>ffa4468a31a8e35dd633f0022ea0b855501c5ba9</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>760.596356</td>\n",
       "      <td>761.847160</td>\n",
       "      <td>758.819721</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2877</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>737.767442</td>\n",
       "      <td>730.038760</td>\n",
       "      <td>729.782946</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2878</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>728.397478</td>\n",
       "      <td>724.251096</td>\n",
       "      <td>723.402412</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2879</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>741.016129</td>\n",
       "      <td>737.701613</td>\n",
       "      <td>737.403226</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21165 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   jun  2017     auto   \n",
       "1     0024f0f900a4ff6923e95a2a20ee94c699ecf167   jun  2017     cool   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   jun  2017     cool   \n",
       "3     0055f438d1ed6195816124e1814d00fe22894e77   jun  2017     cool   \n",
       "4     008b53c11b3b4c56fac3ef938ccd9a34d7755d30   jun  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2875  ff5c81c4eacfb17ff63cecf31c131332abc79ddc   jun  2021     cool   \n",
       "2876  ffa4468a31a8e35dd633f0022ea0b855501c5ba9   jun  2021     cool   \n",
       "2877  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   jun  2021     cool   \n",
       "2878  fffcd8322781a8483039a10c259725e9da9cfdec   jun  2021     cool   \n",
       "2879  ffff46e404a9831c28d8be1a3e427a4377690c6b   jun  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            775.492308   \n",
       "1             auto         Miami            799.983353   \n",
       "2             hold         Miami            796.289140   \n",
       "3             hold         Tampa            753.254545   \n",
       "4             auto         Davie            739.529412   \n",
       "...            ...           ...                   ...   \n",
       "2875          hold    Fort Myers            743.019139   \n",
       "2876          hold    Palm Coast            760.596356   \n",
       "2877          hold  Oakland Park            737.767442   \n",
       "2878          hold   North Miami            728.397478   \n",
       "2879          hold       Parrish            741.016129   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      773.423077                   667.163462   \n",
       "1                      804.537455                   788.787158   \n",
       "2                      806.028507                   790.000000   \n",
       "3                      750.000000                   750.000000   \n",
       "4                      740.000000                   690.000000   \n",
       "...                           ...                          ...   \n",
       "2875                   739.555024                   652.000000   \n",
       "2876                   761.847160                   758.819721   \n",
       "2877                   730.038760                   729.782946   \n",
       "2878                   724.251096                   723.402412   \n",
       "2879                   737.701613                   737.403226   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                    25.0             False         False             True  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2875                 10.0             False         False            False  \n",
       "2876                 10.0             False         False             True  \n",
       "2877                 30.0             False         False            False  \n",
       "2878                 70.0             False         False            False  \n",
       "2879                  0.0             False         False             True  \n",
       "\n",
       "[21165 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/jun/\" + file)\n",
    "    FL_jun = pd.concat([FL_jun, df])\n",
    "    \n",
    "FL_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_jun.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/FL-day/2017-jul-day-FL.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>73f0a3c5962182f20265966172e684d09fb8f462</td>\n",
       "      <td>2017-07-16 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>747</td>\n",
       "      <td>750</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Sunrise</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1412e97d0de077f4ea9ba1ddcf5fc5b6a908c57a</td>\n",
       "      <td>2017-07-18 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>799</td>\n",
       "      <td>800</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>sunrise</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3b02ebbb3b2f2c288629046a9e1e644b523703ac</td>\n",
       "      <td>2017-07-21 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Fort myers</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22a792ab3133c33c7d1606a3d34a71fe122ab43b</td>\n",
       "      <td>2017-07-17 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>790</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Zephyrhills</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ce21d76a1d4bb9340435aa4d9d28a3a1c786d979</td>\n",
       "      <td>2017-07-18 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>772</td>\n",
       "      <td>772</td>\n",
       "      <td>FL</td>\n",
       "      <td>Edgewood</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947958</th>\n",
       "      <td>ba205797d0b02efeebc04a38559eff425fd64f11</td>\n",
       "      <td>2017-07-10 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947959</th>\n",
       "      <td>ba205797d0b02efeebc04a38559eff425fd64f11</td>\n",
       "      <td>2017-07-18 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947960</th>\n",
       "      <td>9a9c02c01d1723f206201e4967e96eac9533c6eb</td>\n",
       "      <td>2017-07-26 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947961</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-07-21 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1947962</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-07-25 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>735</td>\n",
       "      <td>735</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1872241 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        73f0a3c5962182f20265966172e684d09fb8f462  2017-07-16 16:55:00 UTC   \n",
       "1        1412e97d0de077f4ea9ba1ddcf5fc5b6a908c57a  2017-07-18 18:15:00 UTC   \n",
       "2        3b02ebbb3b2f2c288629046a9e1e644b523703ac  2017-07-21 16:45:00 UTC   \n",
       "3        22a792ab3133c33c7d1606a3d34a71fe122ab43b  2017-07-17 11:00:00 UTC   \n",
       "4        ce21d76a1d4bb9340435aa4d9d28a3a1c786d979  2017-07-18 14:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1947958  ba205797d0b02efeebc04a38559eff425fd64f11  2017-07-10 18:25:00 UTC   \n",
       "1947959  ba205797d0b02efeebc04a38559eff425fd64f11  2017-07-18 19:40:00 UTC   \n",
       "1947960  9a9c02c01d1723f206201e4967e96eac9533c6eb  2017-07-26 13:20:00 UTC   \n",
       "1947961  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-07-21 15:30:00 UTC   \n",
       "1947962  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-07-25 17:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               747                      750   \n",
       "1           cool          auto               799                      800   \n",
       "2           cool          auto               783                      780   \n",
       "3           cool          auto               783                      790   \n",
       "4           cool          hold               769                      772   \n",
       "...          ...           ...               ...                      ...   \n",
       "1947958     cool          hold               774                      770   \n",
       "1947959     cool          hold               765                      770   \n",
       "1947960     cool          auto               758                      760   \n",
       "1947961     cool          hold               724                      755   \n",
       "1947962     cool          hold               742                      735   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            650            FL             Sunrise   \n",
       "1                            720            FL             sunrise   \n",
       "2                            760            FL          Fort myers   \n",
       "3                            680            FL         Zephyrhills   \n",
       "4                            772            FL            Edgewood   \n",
       "...                          ...           ...                 ...   \n",
       "1947958                      770            FL  Palm Beach Gardens   \n",
       "1947959                      770            FL  Palm Beach Gardens   \n",
       "1947960                      760            FL  Palm Beach Gardens   \n",
       "1947961                      755            FL  Palm Beach Gardens   \n",
       "1947962                      735            FL  Palm Beach Gardens   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         50             False         False            False   \n",
       "1                         40              True         False            False   \n",
       "2                          7             False         False            False   \n",
       "3                         35             False         False            False   \n",
       "4                         10             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "1947958                   16             False         False            False   \n",
       "1947959                   16             False         False            False   \n",
       "1947960                   15             False         False            False   \n",
       "1947961                    0             False         False            False   \n",
       "1947962                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "1947958                       Gas  \n",
       "1947959                       Gas  \n",
       "1947960                       Gas  \n",
       "1947961                       Gas  \n",
       "1947962                       Gas  \n",
       "\n",
       "[1872241 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/FL/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/FL-day/2018-jul-day-FL.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2fb7284f2f93c5d760a0dac3c99d3bffb377ef81</td>\n",
       "      <td>2018-07-09 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>802</td>\n",
       "      <td>810</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cove Springs</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c8f7ca38deb29189a384d1257d26fe9e26e7ac34</td>\n",
       "      <td>2018-07-28 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>122666318d2b0c22e41ec1d2371ebfa8426c1701</td>\n",
       "      <td>2018-07-12 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>91e8d75c9dfae8004f66f6dfac4e1a1a8b1e5c02</td>\n",
       "      <td>2018-07-15 10:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>de3ef8dc674989be2d2b4a80c74ee9a05b35e09a</td>\n",
       "      <td>2018-07-19 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>750</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>West Miami</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213564</th>\n",
       "      <td>fc80700b84027c9e1b092d52b83a5dd558f2e33f</td>\n",
       "      <td>2018-07-25 07:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>725</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>destin</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213565</th>\n",
       "      <td>3b6ce41439420869b05344a7b300caa95a23b08b</td>\n",
       "      <td>2018-07-04 11:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>2381 oak leaf lane</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213566</th>\n",
       "      <td>151dd2d88561ecf24eab86c85c78d698ff513d71</td>\n",
       "      <td>2018-07-18 19:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>West Palm Beach</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213567</th>\n",
       "      <td>b17d8e97a51cd789660aa23ff62b568ef73700b8</td>\n",
       "      <td>2018-07-13 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4213568</th>\n",
       "      <td>638f66cf0f72b3c82f9dc70c7d8de3c9392abe25</td>\n",
       "      <td>2018-07-18 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4012263 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        2fb7284f2f93c5d760a0dac3c99d3bffb377ef81  2018-07-09 16:05:00 UTC   \n",
       "2        c8f7ca38deb29189a384d1257d26fe9e26e7ac34  2018-07-28 13:35:00 UTC   \n",
       "3        122666318d2b0c22e41ec1d2371ebfa8426c1701  2018-07-12 13:25:00 UTC   \n",
       "4        91e8d75c9dfae8004f66f6dfac4e1a1a8b1e5c02  2018-07-15 10:25:00 UTC   \n",
       "5        de3ef8dc674989be2d2b4a80c74ee9a05b35e09a  2018-07-19 12:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4213564  fc80700b84027c9e1b092d52b83a5dd558f2e33f  2018-07-25 07:25:00 UTC   \n",
       "4213565  3b6ce41439420869b05344a7b300caa95a23b08b  2018-07-04 11:05:00 UTC   \n",
       "4213566  151dd2d88561ecf24eab86c85c78d698ff513d71  2018-07-18 19:20:00 UTC   \n",
       "4213567  b17d8e97a51cd789660aa23ff62b568ef73700b8  2018-07-13 19:15:00 UTC   \n",
       "4213568  638f66cf0f72b3c82f9dc70c7d8de3c9392abe25  2018-07-18 18:20:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               802                      810   \n",
       "2           cool          auto               758                      760   \n",
       "3           cool          hold               757                      770   \n",
       "4           cool          auto               698                      700   \n",
       "5           auto          hold               725                      750   \n",
       "...          ...           ...               ...                      ...   \n",
       "4213564     cool          hold               725                      720   \n",
       "4213565     cool          hold               707                      710   \n",
       "4213566     auto          hold               754                      750   \n",
       "4213567     cool          hold               716                      700   \n",
       "4213568     cool          hold               693                      730   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            720            FL  Green Cove Springs   \n",
       "2                            760            FL             Orlando   \n",
       "3                            770            FL             Orlando   \n",
       "4                            700            FL               Miami   \n",
       "5                            640            FL          West Miami   \n",
       "...                          ...           ...                 ...   \n",
       "4213564                      720            FL              destin   \n",
       "4213565                      710            FL  2381 oak leaf lane   \n",
       "4213566                      640            FL     West Palm Beach   \n",
       "4213567                      700            FL              Naples   \n",
       "4213568                      730            FL               Tampa   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         45             False         False             True   \n",
       "2                          0             False         False             True   \n",
       "3                         70             False         False            False   \n",
       "4                         40             False         False            False   \n",
       "5                         70             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "4213564                   30             False         False             True   \n",
       "4213565                   47             False         False             True   \n",
       "4213566                   40             False         False            False   \n",
       "4213567                    0             False         False            False   \n",
       "4213568                   40              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "4213564                  Electric  \n",
       "4213565                  Electric  \n",
       "4213566                       Gas  \n",
       "4213567                       Gas  \n",
       "4213568                  Electric  \n",
       "\n",
       "[4012263 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/FL/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/FL-day/2019-jul-day-FL.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>be78c32c6828006e4dd1af96d3a17320191c4ece</td>\n",
       "      <td>2019-07-19 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>816</td>\n",
       "      <td>800</td>\n",
       "      <td>640</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2ba34551f430515a665a2f51d698dc35fcd7d99f</td>\n",
       "      <td>2019-07-20 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>782</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Johns</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d39fe584087f62be43472f0bf49573073a6d0b32</td>\n",
       "      <td>2019-07-21 14:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54766be75907ae230c2ff887356cc08db51a3db0</td>\n",
       "      <td>2019-07-17 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Panama City</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1dc854ef4b2dac72dcb199a2b6a5d91a2b016ca5</td>\n",
       "      <td>2019-07-05 12:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5588636</th>\n",
       "      <td>8c4c6ee89b51e3c97e253d86a0ef2d94d12f801d</td>\n",
       "      <td>2019-07-27 08:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>801</td>\n",
       "      <td>800</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5588637</th>\n",
       "      <td>b551ab618fa40c908a11efbac2dd6d8f419e65f2</td>\n",
       "      <td>2019-07-04 10:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Edgewater</td>\n",
       "      <td>47</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5588638</th>\n",
       "      <td>97188f87a5a801b3f28c4b2a46432229c808beb5</td>\n",
       "      <td>2019-07-15 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>750</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>North Port</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5588639</th>\n",
       "      <td>506127c6f61c2a969ae8a370b846c3d509b22948</td>\n",
       "      <td>2019-07-10 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>720</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Ponte Vedra</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5588640</th>\n",
       "      <td>71e8d4a1fd2f63b144916b550bf8cadd82fd66df</td>\n",
       "      <td>2019-07-30 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>740</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5331982 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        be78c32c6828006e4dd1af96d3a17320191c4ece  2019-07-19 15:40:00 UTC   \n",
       "1        2ba34551f430515a665a2f51d698dc35fcd7d99f  2019-07-20 13:05:00 UTC   \n",
       "2        d39fe584087f62be43472f0bf49573073a6d0b32  2019-07-21 14:00:00 UTC   \n",
       "3        54766be75907ae230c2ff887356cc08db51a3db0  2019-07-17 13:20:00 UTC   \n",
       "4        1dc854ef4b2dac72dcb199a2b6a5d91a2b016ca5  2019-07-05 12:20:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "5588636  8c4c6ee89b51e3c97e253d86a0ef2d94d12f801d  2019-07-27 08:30:00 UTC   \n",
       "5588637  b551ab618fa40c908a11efbac2dd6d8f419e65f2  2019-07-04 10:30:00 UTC   \n",
       "5588638  97188f87a5a801b3f28c4b2a46432229c808beb5  2019-07-15 17:10:00 UTC   \n",
       "5588639  506127c6f61c2a969ae8a370b846c3d509b22948  2019-07-10 12:05:00 UTC   \n",
       "5588640  71e8d4a1fd2f63b144916b550bf8cadd82fd66df  2019-07-30 19:25:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          auto               816                      800   \n",
       "1           cool          hold               782                      780   \n",
       "2           auto          hold               759                      755   \n",
       "3           cool          hold               754                      750   \n",
       "4           cool          auto               736                      730   \n",
       "...          ...           ...               ...                      ...   \n",
       "5588636     cool          auto               801                      800   \n",
       "5588637     cool          auto               754                      750   \n",
       "5588638     cool          hold               757                      750   \n",
       "5588639     cool          hold               727                      720   \n",
       "5588640     auto          hold               745                      740   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            640            FL      Orlando   \n",
       "1                            780            FL  Saint Johns   \n",
       "2                            705            FL       Naples   \n",
       "3                            750            FL  Panama City   \n",
       "4                            730            FL        Miami   \n",
       "...                          ...           ...          ...   \n",
       "5588636                      760            FL      Orlando   \n",
       "5588637                      750            FL    Edgewater   \n",
       "5588638                      710            FL   North Port   \n",
       "5588639                      700            FL  Ponte Vedra   \n",
       "5588640                      670            FL   Wellington   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                          9             False         False             True   \n",
       "2                         10             False         False            False   \n",
       "3                          0              True         False             True   \n",
       "4                         10              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "5588636                    5              True         False             True   \n",
       "5588637                   47             False         False             True   \n",
       "5588638                   15             False         False             True   \n",
       "5588639                    0              True         False             True   \n",
       "5588640                   30              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "5588636                  Electric  \n",
       "5588637                  Electric  \n",
       "5588638                  Electric  \n",
       "5588639                  Electric  \n",
       "5588640                       Gas  \n",
       "\n",
       "[5331982 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/FL/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/FL-day/2020-jul-day-FL.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d941abd96271a0b25eb8e71b53c720f1fb694708</td>\n",
       "      <td>2020-07-17 07:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>696</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>FL</td>\n",
       "      <td>Santa Rosa Beach</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7fb08652ae3c5e0f6736959f05403979a9eb64b9</td>\n",
       "      <td>2020-07-13 14:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>775</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Cory Lake Isles</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d4e158fdd9053d5209bebd6900644018bf2d49d1</td>\n",
       "      <td>2020-07-23 19:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>769</td>\n",
       "      <td>770</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>266c3f9ff6582c41af2ea09fda79ae92dfe9d75d</td>\n",
       "      <td>2020-07-07 16:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>745</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Haven</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5b91753ef08f8ba06a84c2a73b77b16b43f3adad</td>\n",
       "      <td>2020-07-13 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>739</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934672</th>\n",
       "      <td>f09a37802504f385407c8c15709060486d2e236e</td>\n",
       "      <td>2020-07-07 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>39</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934673</th>\n",
       "      <td>5eada567967bc0eeab469ffb6448b096ab36916e</td>\n",
       "      <td>2020-07-29 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Petersburg</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934674</th>\n",
       "      <td>c0472acb00956145e2b27225459610c22c636e98</td>\n",
       "      <td>2020-07-20 11:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tierra Verde</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934675</th>\n",
       "      <td>973dc5326b1caaa1f5e11c59c7d0d8f32643cb8a</td>\n",
       "      <td>2020-07-31 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Bradenton</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4934676</th>\n",
       "      <td>c0f87fef6de13bf9a4b03c1c2a411a4bd83a63a7</td>\n",
       "      <td>2020-07-24 19:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>799</td>\n",
       "      <td>790</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>DeBary</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4743486 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        d941abd96271a0b25eb8e71b53c720f1fb694708  2020-07-17 07:45:00 UTC   \n",
       "1        7fb08652ae3c5e0f6736959f05403979a9eb64b9  2020-07-13 14:55:00 UTC   \n",
       "2        d4e158fdd9053d5209bebd6900644018bf2d49d1  2020-07-23 19:00:00 UTC   \n",
       "3        266c3f9ff6582c41af2ea09fda79ae92dfe9d75d  2020-07-07 16:20:00 UTC   \n",
       "4        5b91753ef08f8ba06a84c2a73b77b16b43f3adad  2020-07-13 13:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4934672  f09a37802504f385407c8c15709060486d2e236e  2020-07-07 18:55:00 UTC   \n",
       "4934673  5eada567967bc0eeab469ffb6448b096ab36916e  2020-07-29 18:50:00 UTC   \n",
       "4934674  c0472acb00956145e2b27225459610c22c636e98  2020-07-20 11:05:00 UTC   \n",
       "4934675  973dc5326b1caaa1f5e11c59c7d0d8f32643cb8a  2020-07-31 19:55:00 UTC   \n",
       "4934676  c0f87fef6de13bf9a4b03c1c2a411a4bd83a63a7  2020-07-24 19:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               696                      690   \n",
       "1           cool          hold               779                      775   \n",
       "2           auto          auto               769                      770   \n",
       "3           cool          hold               745                      750   \n",
       "4           cool          auto               739                      780   \n",
       "...          ...           ...               ...                      ...   \n",
       "4934672     cool          auto               729                      730   \n",
       "4934673     cool          hold               702                      700   \n",
       "4934674     auto          auto               756                      755   \n",
       "4934675     cool          hold               712                      720   \n",
       "4934676     auto          hold               799                      790   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            690            FL    Santa Rosa Beach   \n",
       "1                            740            FL     Cory Lake Isles   \n",
       "2                            670            FL               Miami   \n",
       "3                            750            FL        Winter Haven   \n",
       "4                            780            FL  Palm Beach Gardens   \n",
       "...                          ...           ...                 ...   \n",
       "4934672                      730            FL               Tampa   \n",
       "4934673                      700            FL    Saint Petersburg   \n",
       "4934674                      705            FL        Tierra Verde   \n",
       "4934675                      720            FL           Bradenton   \n",
       "4934676                      700            FL              DeBary   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30              True         False            False   \n",
       "1                         15             False         False            False   \n",
       "2                         10              True         False            False   \n",
       "3                         99              True         False             True   \n",
       "4                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "4934672                   39              True         False             True   \n",
       "4934673                   57             False         False            False   \n",
       "4934674                   30              True         False            False   \n",
       "4934675                    0              True         False            False   \n",
       "4934676                   15             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "4934672                  Electric  \n",
       "4934673                       Gas  \n",
       "4934674                       Gas  \n",
       "4934675                       Gas  \n",
       "4934676                  Electric  \n",
       "\n",
       "[4743486 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/FL/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/FL-day/2021-jul-day-FL.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>05a76d6c54bfa59df08600271401ebb1d31190b3</td>\n",
       "      <td>2021-07-08 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>669</td>\n",
       "      <td>660</td>\n",
       "      <td>702</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40cc45fee6a7b9fcc56d490324cb9dd6c1432e71</td>\n",
       "      <td>2021-07-02 11:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>27</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>39bbeab58bc1578fb593378a6fc8a1c2e37a2ae6</td>\n",
       "      <td>2021-07-17 10:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>799</td>\n",
       "      <td>795</td>\n",
       "      <td>615</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61ed265e3dfe33e5c80c3db726dacce4eb68200b</td>\n",
       "      <td>2021-07-15 16:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>770</td>\n",
       "      <td>771</td>\n",
       "      <td>721</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>831d0c3a02456dcd3e7a4df74c3a8cfccf57ec23</td>\n",
       "      <td>2021-07-07 15:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Springs</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2712525</th>\n",
       "      <td>f7e613e6e3201075b5d6012f67b0b199524dc14b</td>\n",
       "      <td>2021-07-24 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>West Palm Beach</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2712526</th>\n",
       "      <td>1a815f301521c8455867939a47b48a642409be3e</td>\n",
       "      <td>2021-07-25 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miramar</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2712527</th>\n",
       "      <td>7b9589fae3891f462eb1004a47bc337193fcb11f</td>\n",
       "      <td>2021-07-17 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Coral Gables</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2712528</th>\n",
       "      <td>a49c183933e6fbb8785004747fd2fa0a63cce070</td>\n",
       "      <td>2021-07-09 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>771</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>45</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2712529</th>\n",
       "      <td>1a815f301521c8455867939a47b48a642409be3e</td>\n",
       "      <td>2021-07-24 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miramar</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2598692 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        05a76d6c54bfa59df08600271401ebb1d31190b3  2021-07-08 18:50:00 UTC   \n",
       "1        40cc45fee6a7b9fcc56d490324cb9dd6c1432e71  2021-07-02 11:00:00 UTC   \n",
       "2        39bbeab58bc1578fb593378a6fc8a1c2e37a2ae6  2021-07-17 10:20:00 UTC   \n",
       "3        61ed265e3dfe33e5c80c3db726dacce4eb68200b  2021-07-15 16:50:00 UTC   \n",
       "4        831d0c3a02456dcd3e7a4df74c3a8cfccf57ec23  2021-07-07 15:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2712525  f7e613e6e3201075b5d6012f67b0b199524dc14b  2021-07-24 10:50:00 UTC   \n",
       "2712526  1a815f301521c8455867939a47b48a642409be3e  2021-07-25 17:20:00 UTC   \n",
       "2712527  7b9589fae3891f462eb1004a47bc337193fcb11f  2021-07-17 15:30:00 UTC   \n",
       "2712528  a49c183933e6fbb8785004747fd2fa0a63cce070  2021-07-09 13:55:00 UTC   \n",
       "2712529  1a815f301521c8455867939a47b48a642409be3e  2021-07-24 16:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               669                      660   \n",
       "1           auto          hold               752                      755   \n",
       "2           auto          hold               799                      795   \n",
       "3           auto          hold               770                      771   \n",
       "4           cool          hold               716                      699   \n",
       "...          ...           ...               ...                      ...   \n",
       "2712525     cool          hold               754                      760   \n",
       "2712526     cool          hold               755                      760   \n",
       "2712527     cool          hold               753                      760   \n",
       "2712528     cool          hold               771                      760   \n",
       "2712529     cool          hold               758                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            702            FL            Miami   \n",
       "1                            705            FL       Palm Coast   \n",
       "2                            615            FL       Wellington   \n",
       "3                            721            FL          Orlando   \n",
       "4                            699            FL    Coral Springs   \n",
       "...                          ...           ...              ...   \n",
       "2712525                      760            FL  West Palm Beach   \n",
       "2712526                      760            FL          Miramar   \n",
       "2712527                      760            FL     Coral Gables   \n",
       "2712528                      760            FL            Miami   \n",
       "2712529                      760            FL          Miramar   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         27              True         False             True   \n",
       "2                         30             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "4                          9             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2712525                   19             False         False            False   \n",
       "2712526                   20             False         False            False   \n",
       "2712527                   60             False         False            False   \n",
       "2712528                   45              True         False            False   \n",
       "2712529                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2712525                       Gas  \n",
       "2712526                       Gas  \n",
       "2712527                       Gas  \n",
       "2712528                       Gas  \n",
       "2712529                       Gas  \n",
       "\n",
       "[2598692 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/FL/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>784.911854</td>\n",
       "      <td>777.190729</td>\n",
       "      <td>677.597264</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>814.774595</td>\n",
       "      <td>815.139465</td>\n",
       "      <td>782.276668</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>796.202381</td>\n",
       "      <td>796.194444</td>\n",
       "      <td>788.253968</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>745.952381</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>740.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>008b53c11b3b4c56fac3ef938ccd9a34d7755d30</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Davie</td>\n",
       "      <td>752.895141</td>\n",
       "      <td>735.759591</td>\n",
       "      <td>735.759591</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2821</th>\n",
       "      <td>ff5c81c4eacfb17ff63cecf31c131332abc79ddc</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Myers</td>\n",
       "      <td>742.608025</td>\n",
       "      <td>738.635802</td>\n",
       "      <td>651.953704</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2822</th>\n",
       "      <td>ffa4468a31a8e35dd633f0022ea0b855501c5ba9</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>760.157003</td>\n",
       "      <td>760.793128</td>\n",
       "      <td>758.903842</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2823</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>755.528256</td>\n",
       "      <td>751.933661</td>\n",
       "      <td>751.859951</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2824</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>731.895143</td>\n",
       "      <td>728.280648</td>\n",
       "      <td>727.478026</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2825</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>740.985075</td>\n",
       "      <td>740.791045</td>\n",
       "      <td>715.477612</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21348 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   jul  2017     auto   \n",
       "1     0024f0f900a4ff6923e95a2a20ee94c699ecf167   jul  2017     cool   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   jul  2017     cool   \n",
       "3     0055f438d1ed6195816124e1814d00fe22894e77   jul  2017     cool   \n",
       "4     008b53c11b3b4c56fac3ef938ccd9a34d7755d30   jul  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2821  ff5c81c4eacfb17ff63cecf31c131332abc79ddc   jul  2021     cool   \n",
       "2822  ffa4468a31a8e35dd633f0022ea0b855501c5ba9   jul  2021     cool   \n",
       "2823  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   jul  2021     cool   \n",
       "2824  fffcd8322781a8483039a10c259725e9da9cfdec   jul  2021     cool   \n",
       "2825  ffff46e404a9831c28d8be1a3e427a4377690c6b   jul  2021     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            784.911854   \n",
       "1             auto         Miami            814.774595   \n",
       "2             hold         Miami            796.202381   \n",
       "3             hold         Tampa            745.952381   \n",
       "4             hold         Davie            752.895141   \n",
       "...            ...           ...                   ...   \n",
       "2821          hold    Fort Myers            742.608025   \n",
       "2822          hold    Palm Coast            760.157003   \n",
       "2823          hold  Oakland Park            755.528256   \n",
       "2824          hold   North Miami            731.895143   \n",
       "2825          hold       Parrish            740.985075   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      777.190729                   677.597264   \n",
       "1                      815.139465                   782.276668   \n",
       "2                      796.194444                   788.253968   \n",
       "3                      740.000000                   740.000000   \n",
       "4                      735.759591                   735.759591   \n",
       "...                           ...                          ...   \n",
       "2821                   738.635802                   651.953704   \n",
       "2822                   760.793128                   758.903842   \n",
       "2823                   751.933661                   751.859951   \n",
       "2824                   728.280648                   727.478026   \n",
       "2825                   740.791045                   715.477612   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                    25.0             False         False             True  \n",
       "4                    30.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2821                 10.0             False         False            False  \n",
       "2822                 10.0             False         False             True  \n",
       "2823                 30.0             False         False            False  \n",
       "2824                 70.0             False         False            False  \n",
       "2825                  0.0             False         False             True  \n",
       "\n",
       "[21348 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/jul/\" + file)\n",
    "    FL_jul = pd.concat([FL_jul, df])\n",
    "    \n",
    "FL_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_jul.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/FL-day/2017-aug-day-FL.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a5ea43460c16b99656551a6e5781d51bf1b58973</td>\n",
       "      <td>2017-08-28 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>793</td>\n",
       "      <td>790</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Apollo Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f6a898aff62ba73d411e8837e264820628af98bd</td>\n",
       "      <td>2017-08-05 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>776</td>\n",
       "      <td>740</td>\n",
       "      <td>620</td>\n",
       "      <td>FL</td>\n",
       "      <td>Hialeah</td>\n",
       "      <td>36</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4239552ec055619245ecff172cb201b0b9a92a7f</td>\n",
       "      <td>2017-08-21 12:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lynn Haven</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2aea9292b9344bb7ab3df64553aee723f78138f3</td>\n",
       "      <td>2017-08-26 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>783</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Deerfield Beach</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22a792ab3133c33c7d1606a3d34a71fe122ab43b</td>\n",
       "      <td>2017-08-17 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>781</td>\n",
       "      <td>790</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Zephyrhills</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062789</th>\n",
       "      <td>b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd</td>\n",
       "      <td>2017-08-02 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062790</th>\n",
       "      <td>ba205797d0b02efeebc04a38559eff425fd64f11</td>\n",
       "      <td>2017-08-29 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062791</th>\n",
       "      <td>ba205797d0b02efeebc04a38559eff425fd64f11</td>\n",
       "      <td>2017-08-18 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>784</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062792</th>\n",
       "      <td>9a9c02c01d1723f206201e4967e96eac9533c6eb</td>\n",
       "      <td>2017-08-20 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2062793</th>\n",
       "      <td>9a9c02c01d1723f206201e4967e96eac9533c6eb</td>\n",
       "      <td>2017-08-29 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>786</td>\n",
       "      <td>780</td>\n",
       "      <td>780</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm Beach Gardens</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1968710 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a5ea43460c16b99656551a6e5781d51bf1b58973  2017-08-28 19:25:00 UTC   \n",
       "1        f6a898aff62ba73d411e8837e264820628af98bd  2017-08-05 17:25:00 UTC   \n",
       "2        4239552ec055619245ecff172cb201b0b9a92a7f  2017-08-21 12:20:00 UTC   \n",
       "3        2aea9292b9344bb7ab3df64553aee723f78138f3  2017-08-26 18:00:00 UTC   \n",
       "4        22a792ab3133c33c7d1606a3d34a71fe122ab43b  2017-08-17 13:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2062789  b14f9eab461a06d27bcaf2bb3103d63fc9aefbdd  2017-08-02 15:30:00 UTC   \n",
       "2062790  ba205797d0b02efeebc04a38559eff425fd64f11  2017-08-29 14:10:00 UTC   \n",
       "2062791  ba205797d0b02efeebc04a38559eff425fd64f11  2017-08-18 19:45:00 UTC   \n",
       "2062792  9a9c02c01d1723f206201e4967e96eac9533c6eb  2017-08-20 18:00:00 UTC   \n",
       "2062793  9a9c02c01d1723f206201e4967e96eac9533c6eb  2017-08-29 19:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               793                      790   \n",
       "1           cool          auto               776                      740   \n",
       "2           auto          hold               723                      750   \n",
       "3           cool          hold               783                      780   \n",
       "4           cool          auto               781                      790   \n",
       "...          ...           ...               ...                      ...   \n",
       "2062789     cool          hold               743                      740   \n",
       "2062790     cool          hold               765                      770   \n",
       "2062791     cool          hold               784                      780   \n",
       "2062792     cool          auto               751                      750   \n",
       "2062793     cool          hold               786                      780   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            790            FL        Apollo Beach   \n",
       "1                            620            FL             Hialeah   \n",
       "2                            700            FL          Lynn Haven   \n",
       "3                            780            FL     Deerfield Beach   \n",
       "4                            680            FL         Zephyrhills   \n",
       "...                          ...           ...                 ...   \n",
       "2062789                      740            FL  Palm Beach Gardens   \n",
       "2062790                      770            FL  Palm Beach Gardens   \n",
       "2062791                      780            FL  Palm Beach Gardens   \n",
       "2062792                      740            FL  Palm Beach Gardens   \n",
       "2062793                      780            FL  Palm Beach Gardens   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False             True   \n",
       "1                         36             False         False            False   \n",
       "2                         10             False         False            False   \n",
       "3                         25             False         False            False   \n",
       "4                         35             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2062789                    0             False         False            False   \n",
       "2062790                   16             False         False            False   \n",
       "2062791                   16             False         False            False   \n",
       "2062792                   15             False         False            False   \n",
       "2062793                   15             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2062789                       Gas  \n",
       "2062790                       Gas  \n",
       "2062791                       Gas  \n",
       "2062792                       Gas  \n",
       "2062793                       Gas  \n",
       "\n",
       "[1968710 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/FL/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/FL-day/2018-aug-day-FL.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d2c07331cdcbf3f31657d4c870f6fb57f742e637</td>\n",
       "      <td>2018-08-24 17:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e478562afbe7db9cd8aa42235cb8fb37cc134b92</td>\n",
       "      <td>2018-08-06 10:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>wilton manors</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50acae4c25caed8fa44fc2db5e7d534551094247</td>\n",
       "      <td>2018-08-26 10:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>04c61c67ac8e504d333835cb46d9fddfea22b6af</td>\n",
       "      <td>2018-08-24 16:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Panama City</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8dce0f3d0feffc4c57c1b1ca09e56e072325615d</td>\n",
       "      <td>2018-08-17 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>777</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>New Port Richey</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4200342</th>\n",
       "      <td>a91f348d1f2b7641556a9020c83b32f6c524951b</td>\n",
       "      <td>2018-08-31 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Port Saint Lucie</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4200343</th>\n",
       "      <td>8177357efd7422d3168925039c70508e0d2f6ffb</td>\n",
       "      <td>2018-08-09 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>750</td>\n",
       "      <td>660</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lehigh Acres</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4200345</th>\n",
       "      <td>829150865af973ebe56f195195814c2c929ac4bc</td>\n",
       "      <td>2018-08-01 18:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lake Clarke Shores</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4200346</th>\n",
       "      <td>49304c734512b21c6ad58cd7c16fe605020d5824</td>\n",
       "      <td>2018-08-08 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>847</td>\n",
       "      <td>842</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4200347</th>\n",
       "      <td>eb058ddb988b7d3a21a5fba57c2a2c7824aae860</td>\n",
       "      <td>2018-08-17 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>768</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oviedo</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4005776 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        d2c07331cdcbf3f31657d4c870f6fb57f742e637  2018-08-24 17:35:00 UTC   \n",
       "1        e478562afbe7db9cd8aa42235cb8fb37cc134b92  2018-08-06 10:30:00 UTC   \n",
       "2        50acae4c25caed8fa44fc2db5e7d534551094247  2018-08-26 10:45:00 UTC   \n",
       "3        04c61c67ac8e504d333835cb46d9fddfea22b6af  2018-08-24 16:10:00 UTC   \n",
       "4        8dce0f3d0feffc4c57c1b1ca09e56e072325615d  2018-08-17 17:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4200342  a91f348d1f2b7641556a9020c83b32f6c524951b  2018-08-31 18:15:00 UTC   \n",
       "4200343  8177357efd7422d3168925039c70508e0d2f6ffb  2018-08-09 19:25:00 UTC   \n",
       "4200345  829150865af973ebe56f195195814c2c929ac4bc  2018-08-01 18:25:00 UTC   \n",
       "4200346  49304c734512b21c6ad58cd7c16fe605020d5824  2018-08-08 19:30:00 UTC   \n",
       "4200347  eb058ddb988b7d3a21a5fba57c2a2c7824aae860  2018-08-17 11:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               743                      740   \n",
       "1           cool          hold               736                      760   \n",
       "2           cool          auto               762                      760   \n",
       "3           cool          hold               720                      720   \n",
       "4           cool          auto               777                      780   \n",
       "...          ...           ...               ...                      ...   \n",
       "4200342     cool          auto               766                      760   \n",
       "4200343     auto          hold               753                      750   \n",
       "4200345     cool          hold               712                      710   \n",
       "4200346     cool          hold               847                      842   \n",
       "4200347     cool          auto               768                      770   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            740            FL               Tampa   \n",
       "1                            760            FL      wilton manors    \n",
       "2                            760            FL           Kissimmee   \n",
       "3                            720            FL         Panama City   \n",
       "4                            760            FL     New Port Richey   \n",
       "...                          ...           ...                 ...   \n",
       "4200342                      760            FL    Port Saint Lucie   \n",
       "4200343                      660            FL        Lehigh Acres   \n",
       "4200345                      710            FL  Lake Clarke Shores   \n",
       "4200346                      790            FL             Orlando   \n",
       "4200347                      770            FL              Oviedo   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          5              True         False             True   \n",
       "1                          9              True         False            False   \n",
       "2                         10              True         False             True   \n",
       "3                         20             False         False             True   \n",
       "4                          0             False         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "4200342                   30             False         False            False   \n",
       "4200343                   15             False         False            False   \n",
       "4200345                   20             False         False            False   \n",
       "4200346                   10              True         False            False   \n",
       "4200347                   30              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "4200342                       Gas  \n",
       "4200343                       Gas  \n",
       "4200345                       Gas  \n",
       "4200346                       Gas  \n",
       "4200347                  Electric  \n",
       "\n",
       "[4005776 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/FL/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/FL-day/2019-aug-day-FL.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fe1ed9cc5a1195b930b7a5a4e7b036f23a1d9f9f</td>\n",
       "      <td>2019-08-07 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8aa0adcddcac9c1048e4b25d6571ce91e589b232</td>\n",
       "      <td>2019-08-05 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>771</td>\n",
       "      <td>788</td>\n",
       "      <td>788</td>\n",
       "      <td>FL</td>\n",
       "      <td>Weston</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>45ff9e901fb365c4b300c5b3b70b588f89efa3bd</td>\n",
       "      <td>2019-08-24 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>804</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Brooksville</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>77c062ad98bf705476beb44304454bf06541241c</td>\n",
       "      <td>2019-08-02 13:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>39</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2ab4fe0bece07e7db7efe65b382dc0d4d32adc34</td>\n",
       "      <td>2019-08-07 12:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>746</td>\n",
       "      <td>750</td>\n",
       "      <td>650</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pompano Beach</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5540518</th>\n",
       "      <td>5137642ba2cfc0e1868c3ab0f870bfdc51c5acf4</td>\n",
       "      <td>2019-08-04 12:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>770</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>DeBary</td>\n",
       "      <td>28</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5540519</th>\n",
       "      <td>ac34951fa83804c0ba5fedbfc8b7232799b734fa</td>\n",
       "      <td>2019-08-21 12:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>730</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5540520</th>\n",
       "      <td>abc4c060958b64613d404eebe095083dd33a5455</td>\n",
       "      <td>2019-08-26 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>620</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5540521</th>\n",
       "      <td>30a8c18e1ef2aaddfda13e7efe137e5679801ecc</td>\n",
       "      <td>2019-08-23 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>FL</td>\n",
       "      <td>Panama City Beach</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5540522</th>\n",
       "      <td>1a85f3c1a9e961a13a3072ea3dd272b6c5510f95</td>\n",
       "      <td>2019-08-18 12:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Port Orange</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5294522 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        fe1ed9cc5a1195b930b7a5a4e7b036f23a1d9f9f  2019-08-07 19:15:00 UTC   \n",
       "1        8aa0adcddcac9c1048e4b25d6571ce91e589b232  2019-08-05 14:10:00 UTC   \n",
       "2        45ff9e901fb365c4b300c5b3b70b588f89efa3bd  2019-08-24 17:05:00 UTC   \n",
       "3        77c062ad98bf705476beb44304454bf06541241c  2019-08-02 13:25:00 UTC   \n",
       "4        2ab4fe0bece07e7db7efe65b382dc0d4d32adc34  2019-08-07 12:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "5540518  5137642ba2cfc0e1868c3ab0f870bfdc51c5acf4  2019-08-04 12:10:00 UTC   \n",
       "5540519  ac34951fa83804c0ba5fedbfc8b7232799b734fa  2019-08-21 12:05:00 UTC   \n",
       "5540520  abc4c060958b64613d404eebe095083dd33a5455  2019-08-26 18:35:00 UTC   \n",
       "5540521  30a8c18e1ef2aaddfda13e7efe137e5679801ecc  2019-08-23 13:50:00 UTC   \n",
       "5540522  1a85f3c1a9e961a13a3072ea3dd272b6c5510f95  2019-08-18 12:45:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               717                      740   \n",
       "1           cool          auto               771                      788   \n",
       "2           cool          auto               804                      800   \n",
       "3           cool          auto               742                      760   \n",
       "4           auto          auto               746                      750   \n",
       "...          ...           ...               ...                      ...   \n",
       "5540518     auto          hold               756                      770   \n",
       "5540519     auto          hold               722                      730   \n",
       "5540520     cool          auto               751                      750   \n",
       "5540521     cool          hold               714                      715   \n",
       "5540522     cool          auto               723                      720   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            740            FL             Naples   \n",
       "1                            788            FL             Weston   \n",
       "2                            790            FL        Brooksville   \n",
       "3                            760            FL              Tampa   \n",
       "4                            650            FL      Pompano Beach   \n",
       "...                          ...           ...                ...   \n",
       "5540518                      680            FL             DeBary   \n",
       "5540519                      680            FL         Wellington   \n",
       "5540520                      620            FL              Miami   \n",
       "5540521                      715            FL  Panama City Beach   \n",
       "5540522                      720            FL        Port Orange   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20              True         False            False   \n",
       "1                         15             False         False            False   \n",
       "2                         20             False         False             True   \n",
       "3                         39             False         False            False   \n",
       "4                          9              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "5540518                   28              True         False             True   \n",
       "5540519                   25             False         False            False   \n",
       "5540520                    0              True         False            False   \n",
       "5540521                   19              True         False            False   \n",
       "5540522                    0             False         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "5540518                  Electric  \n",
       "5540519                       Gas  \n",
       "5540520                       Gas  \n",
       "5540521                       Gas  \n",
       "5540522                  Electric  \n",
       "\n",
       "[5294522 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/FL/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/FL-day/2020-aug-day-FL.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d4c177a4503855591f0adc0bab3ed9e4b30d6d1c</td>\n",
       "      <td>2020-08-29 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Port St Lucie</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>896739e4214ffa8a8d39fa7de7077d95a1f503d1</td>\n",
       "      <td>2020-08-21 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Royal Palm Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8744d57ebb50663b3c41af3614e8c445044d844a</td>\n",
       "      <td>2020-08-03 09:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Green Cove Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d6c2e44b80e82e437e07bc818b818327a926c101</td>\n",
       "      <td>2020-08-20 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>783</td>\n",
       "      <td>780</td>\n",
       "      <td>720</td>\n",
       "      <td>FL</td>\n",
       "      <td>Homestead</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3dd58b7226a0e7f9bd4ed9f1b1ca976b5388d60d</td>\n",
       "      <td>2020-08-27 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>725</td>\n",
       "      <td>720</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Doram</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995721</th>\n",
       "      <td>2bab5529487e7ac80bf4d34f169a9d908e6ec7b2</td>\n",
       "      <td>2020-08-12 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>674</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Freeport</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995722</th>\n",
       "      <td>7603fd4aa87d06e1e13680bdc2ed797c95d1776c</td>\n",
       "      <td>2020-08-22 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>820</td>\n",
       "      <td>820</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Delray Beach</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995723</th>\n",
       "      <td>79ebc57aaa45e453673eeb53cc3d55e6395937ac</td>\n",
       "      <td>2020-08-23 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Delray Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995724</th>\n",
       "      <td>a8b12ab27e5fd886a42b40f7b865429b895ef384</td>\n",
       "      <td>2020-08-04 07:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Park</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995725</th>\n",
       "      <td>78c544d449e0387edcff821adcda02ed6712c8ea</td>\n",
       "      <td>2020-08-12 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>774</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Venice</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4774986 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        d4c177a4503855591f0adc0bab3ed9e4b30d6d1c  2020-08-29 17:25:00 UTC   \n",
       "1        896739e4214ffa8a8d39fa7de7077d95a1f503d1  2020-08-21 17:50:00 UTC   \n",
       "2        8744d57ebb50663b3c41af3614e8c445044d844a  2020-08-03 09:20:00 UTC   \n",
       "3        d6c2e44b80e82e437e07bc818b818327a926c101  2020-08-20 18:40:00 UTC   \n",
       "4        3dd58b7226a0e7f9bd4ed9f1b1ca976b5388d60d  2020-08-27 13:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4995721  2bab5529487e7ac80bf4d34f169a9d908e6ec7b2  2020-08-12 12:10:00 UTC   \n",
       "4995722  7603fd4aa87d06e1e13680bdc2ed797c95d1776c  2020-08-22 14:30:00 UTC   \n",
       "4995723  79ebc57aaa45e453673eeb53cc3d55e6395937ac  2020-08-23 17:50:00 UTC   \n",
       "4995724  a8b12ab27e5fd886a42b40f7b865429b895ef384  2020-08-04 07:30:00 UTC   \n",
       "4995725  78c544d449e0387edcff821adcda02ed6712c8ea  2020-08-12 19:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               764                      760   \n",
       "1           cool          auto               718                      680   \n",
       "2           cool          hold               727                      730   \n",
       "3           auto          hold               783                      780   \n",
       "4           cool          auto               725                      720   \n",
       "...          ...           ...               ...                      ...   \n",
       "4995721     cool          auto               674                      680   \n",
       "4995722     cool          auto               820                      820   \n",
       "4995723     auto          hold               740                      740   \n",
       "4995724     cool          auto               741                      740   \n",
       "4995725     cool          auto               774                      770   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            760            FL       Port St Lucie   \n",
       "1                            680            FL    Royal Palm Beach   \n",
       "2                            730            FL  Green Cove Springs   \n",
       "3                            720            FL           Homestead   \n",
       "4                            740            FL               Doram   \n",
       "...                          ...           ...                 ...   \n",
       "4995721                      680            FL            Freeport   \n",
       "4995722                      790            FL        Delray Beach   \n",
       "4995723                      670            FL        Delray Beach   \n",
       "4995724                      740            FL         Winter Park   \n",
       "4995725                      770            FL              Venice   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         15             False         False            False   \n",
       "1                          0              True         False            False   \n",
       "2                          0              True         False             True   \n",
       "3                          5             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "4995721                    0             False         False             True   \n",
       "4995722                   40             False         False            False   \n",
       "4995723                    0             False         False            False   \n",
       "4995724                   60             False         False            False   \n",
       "4995725                   20              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "4995721                  Electric  \n",
       "4995722                       Gas  \n",
       "4995723                       Gas  \n",
       "4995724                       Gas  \n",
       "4995725                  Electric  \n",
       "\n",
       "[4774986 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/FL/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>782.218335</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>702.132690</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>816.357640</td>\n",
       "      <td>817.450225</td>\n",
       "      <td>784.571897</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>802.366295</td>\n",
       "      <td>803.643454</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>735.250000</td>\n",
       "      <td>734.833333</td>\n",
       "      <td>660.583333</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>750.834862</td>\n",
       "      <td>742.440367</td>\n",
       "      <td>741.862385</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5391</th>\n",
       "      <td>ff65db3b80b5965987eb40d20a79a3a0b8045a07</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ponte Vedra Beach</td>\n",
       "      <td>758.774194</td>\n",
       "      <td>760.129032</td>\n",
       "      <td>640.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5392</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>728.743682</td>\n",
       "      <td>713.379061</td>\n",
       "      <td>712.660650</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5393</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>725.470588</td>\n",
       "      <td>714.188971</td>\n",
       "      <td>713.086029</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5394</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>752.978723</td>\n",
       "      <td>750.085106</td>\n",
       "      <td>749.716312</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5395</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>758.731707</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18528 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   aug  2017     auto   \n",
       "1     0024f0f900a4ff6923e95a2a20ee94c699ecf167   aug  2017     cool   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   aug  2017     cool   \n",
       "3     0055f438d1ed6195816124e1814d00fe22894e77   aug  2017     cool   \n",
       "4     0055f438d1ed6195816124e1814d00fe22894e77   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "5391  ff65db3b80b5965987eb40d20a79a3a0b8045a07   aug  2020     cool   \n",
       "5392  fffcd8322781a8483039a10c259725e9da9cfdec   aug  2020     cool   \n",
       "5393  fffcd8322781a8483039a10c259725e9da9cfdec   aug  2020     cool   \n",
       "5394  ffff46e404a9831c28d8be1a3e427a4377690c6b   aug  2020     cool   \n",
       "5395  ffff46e404a9831c28d8be1a3e427a4377690c6b   aug  2020     cool   \n",
       "\n",
       "     CalendarEvent               City  Temperature_ctrl_ave  \\\n",
       "0             auto       Jacksonville            782.218335   \n",
       "1             auto              Miami            816.357640   \n",
       "2             hold              Miami            802.366295   \n",
       "3             auto              Tampa            735.250000   \n",
       "4             hold              Tampa            750.834862   \n",
       "...            ...                ...                   ...   \n",
       "5391          auto  Ponte Vedra Beach            758.774194   \n",
       "5392          auto        North Miami            728.743682   \n",
       "5393          hold        North Miami            725.470588   \n",
       "5394          auto            Parrish            752.978723   \n",
       "5395          hold            Parrish            758.731707   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      780.000000                   702.132690   \n",
       "1                      817.450225                   784.571897   \n",
       "2                      803.643454                   790.000000   \n",
       "3                      734.833333                   660.583333   \n",
       "4                      742.440367                   741.862385   \n",
       "...                           ...                          ...   \n",
       "5391                   760.129032                   640.000000   \n",
       "5392                   713.379061                   712.660650   \n",
       "5393                   714.188971                   713.086029   \n",
       "5394                   750.085106                   749.716312   \n",
       "5395                   750.000000                   750.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                    25.0             False         False             True  \n",
       "4                    25.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "5391                  0.0              True         False             True  \n",
       "5392                 70.0             False         False            False  \n",
       "5393                 70.0             False         False            False  \n",
       "5394                  0.0             False         False             True  \n",
       "5395                  0.0             False         False             True  \n",
       "\n",
       "[18528 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/aug/\" + file)\n",
    "    FL_aug = pd.concat([FL_aug, df])\n",
    "    \n",
    "FL_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_aug.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/FL-day/2017-dec-day-FL.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2aca1298389867e8ccfeea99c00820cdabe5efaf</td>\n",
       "      <td>2017-12-03 16:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>FL</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>66</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4b5da78a3b30b5e6d3aaa6bf5d2cd9969553788e</td>\n",
       "      <td>2017-12-19 08:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>725</td>\n",
       "      <td>725</td>\n",
       "      <td>FL</td>\n",
       "      <td>Cape Coral</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>91385f63c632a2f4c17d0ebbc8cbae4099adfaaa</td>\n",
       "      <td>2017-12-07 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>755</td>\n",
       "      <td>755</td>\n",
       "      <td>FL</td>\n",
       "      <td>Clearwater</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7590e23eefb20bb153a55276567d6e6e9c59b3a8</td>\n",
       "      <td>2017-12-29 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>715</td>\n",
       "      <td>770</td>\n",
       "      <td>749</td>\n",
       "      <td>FL</td>\n",
       "      <td>Largo</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2bdd8511d93f1f0d9847002b03864e279b8927b9</td>\n",
       "      <td>2017-12-14 18:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>775</td>\n",
       "      <td>725</td>\n",
       "      <td>FL</td>\n",
       "      <td>Kissimmee</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2532124</th>\n",
       "      <td>9803f7459c87eefe0f767aa55ba0e50bbc433d4f</td>\n",
       "      <td>2017-12-05 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Surfside</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2532125</th>\n",
       "      <td>0e57c47d25177db6398cd2261703eae74453527b</td>\n",
       "      <td>2017-12-26 12:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Petersburg</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2532126</th>\n",
       "      <td>357f5810804e474cf533893fbfadd15eae784ab3</td>\n",
       "      <td>2017-12-23 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>777</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Fort Lauderdale</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2532127</th>\n",
       "      <td>8a188da39be9d3c57e82aee48ec5d5b3f6384c72</td>\n",
       "      <td>2017-12-18 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2532128</th>\n",
       "      <td>5381b3b8be5f1a013a4bf68a3e865a146f344cb1</td>\n",
       "      <td>2017-12-03 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>palm beach gardens</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2429660 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        2aca1298389867e8ccfeea99c00820cdabe5efaf  2017-12-03 16:50:00 UTC   \n",
       "1        4b5da78a3b30b5e6d3aaa6bf5d2cd9969553788e  2017-12-19 08:15:00 UTC   \n",
       "2        91385f63c632a2f4c17d0ebbc8cbae4099adfaaa  2017-12-07 17:05:00 UTC   \n",
       "3        7590e23eefb20bb153a55276567d6e6e9c59b3a8  2017-12-29 17:40:00 UTC   \n",
       "4        2bdd8511d93f1f0d9847002b03864e279b8927b9  2017-12-14 18:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2532124  9803f7459c87eefe0f767aa55ba0e50bbc433d4f  2017-12-05 19:05:00 UTC   \n",
       "2532125  0e57c47d25177db6398cd2261703eae74453527b  2017-12-26 12:20:00 UTC   \n",
       "2532126  357f5810804e474cf533893fbfadd15eae784ab3  2017-12-23 14:25:00 UTC   \n",
       "2532127  8a188da39be9d3c57e82aee48ec5d5b3f6384c72  2017-12-18 13:10:00 UTC   \n",
       "2532128  5381b3b8be5f1a013a4bf68a3e865a146f344cb1  2017-12-03 12:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               734                      729   \n",
       "1           cool          hold               722                      725   \n",
       "2           cool          hold               752                      755   \n",
       "3           cool          auto               715                      770   \n",
       "4           auto          hold               741                      775   \n",
       "...          ...           ...               ...                      ...   \n",
       "2532124     cool          hold               764                      760   \n",
       "2532125     cool          hold               702                      760   \n",
       "2532126     cool          auto               777                      780   \n",
       "2532127     cool          hold               733                      760   \n",
       "2532128     cool          hold               728                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            729            FL        Jacksonville   \n",
       "1                            725            FL          Cape Coral   \n",
       "2                            755            FL          Clearwater   \n",
       "3                            749            FL               Largo   \n",
       "4                            725            FL           Kissimmee   \n",
       "...                          ...           ...                 ...   \n",
       "2532124                      760            FL            Surfside   \n",
       "2532125                      760            FL    Saint Petersburg   \n",
       "2532126                      760            FL     Fort Lauderdale   \n",
       "2532127                      760            FL        Oakland Park   \n",
       "2532128                      760            FL  palm beach gardens   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         66             False         False             True   \n",
       "1                         15             False         False            False   \n",
       "2                          9             False         False             True   \n",
       "3                         16             False         False             True   \n",
       "4                          5              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2532124                   67             False         False            False   \n",
       "2532125                   60             False         False            False   \n",
       "2532126                   10             False         False            False   \n",
       "2532127                   55             False         False            False   \n",
       "2532128                   30             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "2532124                       Gas  \n",
       "2532125                       Gas  \n",
       "2532126                       Gas  \n",
       "2532127                       Gas  \n",
       "2532128                       Gas  \n",
       "\n",
       "[2429660 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/FL/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/FL-day/2018-dec-day-FL.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9d9804ca59adb6c9bcf875fafd4195177c1f42bb</td>\n",
       "      <td>2018-12-22 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Belleview</td>\n",
       "      <td>36</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8345461325a6e244d4ee3a0dbec3b2b6c1876177</td>\n",
       "      <td>2018-12-07 16:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>790</td>\n",
       "      <td>630</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e20a3e765cdf2b7a2626a87af28bd4f0b6587df8</td>\n",
       "      <td>2018-12-07 11:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>770</td>\n",
       "      <td>660</td>\n",
       "      <td>FL</td>\n",
       "      <td>South Pasadena</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13786be965ee366346ca6109b31adf39fffef7e8</td>\n",
       "      <td>2018-12-05 11:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>730</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Apopka</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f785c2313faacc4c9b689c0008adcedd462c51b9</td>\n",
       "      <td>2018-12-10 17:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>717</td>\n",
       "      <td>780</td>\n",
       "      <td>730</td>\n",
       "      <td>FL</td>\n",
       "      <td>Winter Park</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4018653</th>\n",
       "      <td>b4fb7fc34719d3ac281b6bc35f25cb59601be3cb</td>\n",
       "      <td>2018-12-07 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4018654</th>\n",
       "      <td>e0c573fc0956caee9be6df81fadeb66d0685f520</td>\n",
       "      <td>2018-12-04 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>733</td>\n",
       "      <td>735</td>\n",
       "      <td>705</td>\n",
       "      <td>FL</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4018655</th>\n",
       "      <td>aa358226200f2f58c623ffc8beb3f5f9901817a8</td>\n",
       "      <td>2018-12-19 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Gulf Breeze</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4018657</th>\n",
       "      <td>03e91d495e89a9684d399da6baa0890992bf6aa3</td>\n",
       "      <td>2018-12-08 16:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4018658</th>\n",
       "      <td>f8112833516bffb9e17c7721b913ec8bad6873af</td>\n",
       "      <td>2018-12-27 13:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Lehigh Acres</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3818854 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        9d9804ca59adb6c9bcf875fafd4195177c1f42bb  2018-12-22 12:25:00 UTC   \n",
       "1        8345461325a6e244d4ee3a0dbec3b2b6c1876177  2018-12-07 16:35:00 UTC   \n",
       "2        e20a3e765cdf2b7a2626a87af28bd4f0b6587df8  2018-12-07 11:35:00 UTC   \n",
       "3        13786be965ee366346ca6109b31adf39fffef7e8  2018-12-05 11:25:00 UTC   \n",
       "4        f785c2313faacc4c9b689c0008adcedd462c51b9  2018-12-10 17:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4018653  b4fb7fc34719d3ac281b6bc35f25cb59601be3cb  2018-12-07 19:30:00 UTC   \n",
       "4018654  e0c573fc0956caee9be6df81fadeb66d0685f520  2018-12-04 13:55:00 UTC   \n",
       "4018655  aa358226200f2f58c623ffc8beb3f5f9901817a8  2018-12-19 19:15:00 UTC   \n",
       "4018657  03e91d495e89a9684d399da6baa0890992bf6aa3  2018-12-08 16:35:00 UTC   \n",
       "4018658  f8112833516bffb9e17c7721b913ec8bad6873af  2018-12-27 13:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               681                      760   \n",
       "1           auto          hold               772                      790   \n",
       "2           auto          hold               679                      770   \n",
       "3           cool          auto               706                      730   \n",
       "4           auto          auto               717                      780   \n",
       "...          ...           ...               ...                      ...   \n",
       "4018653     cool          hold               717                      770   \n",
       "4018654     auto          hold               733                      735   \n",
       "4018655     heat          auto               697                      700   \n",
       "4018657     heat          hold               719                      700   \n",
       "4018658     cool          hold               711                      750   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            680            FL       Belleview   \n",
       "1                            630            FL           Miami   \n",
       "2                            660            FL  South Pasadena   \n",
       "3                            730            FL          Apopka   \n",
       "4                            730            FL     Winter Park   \n",
       "...                          ...           ...             ...   \n",
       "4018653                      770            FL         Orlando   \n",
       "4018654                      705            FL    Jacksonville   \n",
       "4018655                      700            FL     Gulf Breeze   \n",
       "4018657                      700            FL           Tampa   \n",
       "4018658                      750            FL    Lehigh Acres   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         36              True         False             True   \n",
       "1                         10             False         False            False   \n",
       "2                         50              True         False            False   \n",
       "3                         40              True         False             True   \n",
       "4                         60              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "4018653                    0              True         False             True   \n",
       "4018654                    0             False         False             True   \n",
       "4018655                   40              True         False             True   \n",
       "4018657                    0              True         False             True   \n",
       "4018658                   20              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "4018653                  Electric  \n",
       "4018654                  Electric  \n",
       "4018655                  Electric  \n",
       "4018657                  Electric  \n",
       "4018658                       Gas  \n",
       "\n",
       "[3818854 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/FL/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/FL-day/2019-dec-day-FL.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9b9ad5b92fe32843968fdcbf0ae83f786bdcb2a2</td>\n",
       "      <td>2019-12-26 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>FL</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0091af859415cf1a74a3666695db239ecfa92c55</td>\n",
       "      <td>2019-12-02 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>701</td>\n",
       "      <td>720</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Altamonte Springs</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>43556c904d05d3fe19d094253d7a09acc42ad61d</td>\n",
       "      <td>2019-12-21 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>664</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miramar Beach</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dda8987a7ceee737a30cd6f25c81e755320f1272</td>\n",
       "      <td>2019-12-27 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>729</td>\n",
       "      <td>730</td>\n",
       "      <td>670</td>\n",
       "      <td>FL</td>\n",
       "      <td>Orlando</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f67a59ac9a3fd1b5ac966b3dd6f77593911b0207</td>\n",
       "      <td>2019-12-07 16:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>FL</td>\n",
       "      <td>Palm City</td>\n",
       "      <td>35</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4415886</th>\n",
       "      <td>7e13a89c5f749bd9972324c45b0ad38f23c992ec</td>\n",
       "      <td>2019-12-08 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>704</td>\n",
       "      <td>820</td>\n",
       "      <td>790</td>\n",
       "      <td>FL</td>\n",
       "      <td>Port Charlotte</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4415887</th>\n",
       "      <td>91c39be38e40567a1043c1c3c00c3ac6a948f5f1</td>\n",
       "      <td>2019-12-25 18:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4415888</th>\n",
       "      <td>28891111b428da1b8a00ffbedee22c44503e44fc</td>\n",
       "      <td>2019-12-15 17:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>745</td>\n",
       "      <td>685</td>\n",
       "      <td>FL</td>\n",
       "      <td>Destin</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4415890</th>\n",
       "      <td>2c378c79ac67770fe44c51241998f7da174f22f4</td>\n",
       "      <td>2019-12-16 09:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>751</td>\n",
       "      <td>751</td>\n",
       "      <td>FL</td>\n",
       "      <td>Wellington</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4415891</th>\n",
       "      <td>49c4c11b8b27e2b888a6a92ae01f4741d4ac69e3</td>\n",
       "      <td>2019-12-02 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>752</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami Beach</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4218327 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        9b9ad5b92fe32843968fdcbf0ae83f786bdcb2a2  2019-12-26 12:50:00 UTC   \n",
       "1        0091af859415cf1a74a3666695db239ecfa92c55  2019-12-02 18:50:00 UTC   \n",
       "2        43556c904d05d3fe19d094253d7a09acc42ad61d  2019-12-21 14:10:00 UTC   \n",
       "3        dda8987a7ceee737a30cd6f25c81e755320f1272  2019-12-27 17:20:00 UTC   \n",
       "4        f67a59ac9a3fd1b5ac966b3dd6f77593911b0207  2019-12-07 16:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "4415886  7e13a89c5f749bd9972324c45b0ad38f23c992ec  2019-12-08 13:45:00 UTC   \n",
       "4415887  91c39be38e40567a1043c1c3c00c3ac6a948f5f1  2019-12-25 18:45:00 UTC   \n",
       "4415888  28891111b428da1b8a00ffbedee22c44503e44fc  2019-12-15 17:45:00 UTC   \n",
       "4415890  2c378c79ac67770fe44c51241998f7da174f22f4  2019-12-16 09:15:00 UTC   \n",
       "4415891  49c4c11b8b27e2b888a6a92ae01f4741d4ac69e3  2019-12-02 14:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               695                      760   \n",
       "1           cool          auto               701                      720   \n",
       "2           cool          auto               664                      770   \n",
       "3           auto          auto               729                      730   \n",
       "4           cool          hold               708                      700   \n",
       "...          ...           ...               ...                      ...   \n",
       "4415886     cool          auto               704                      820   \n",
       "4415887     cool          hold               743                      750   \n",
       "4415888     auto          hold               699                      745   \n",
       "4415890     cool          hold               751                      751   \n",
       "4415891     cool          auto               752                      750   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            680            FL       Jacksonville   \n",
       "1                            700            FL  Altamonte Springs   \n",
       "2                            770            FL      Miramar Beach   \n",
       "3                            670            FL            Orlando   \n",
       "4                            700            FL          Palm City   \n",
       "...                          ...           ...                ...   \n",
       "4415886                      790            FL     Port Charlotte   \n",
       "4415887                      750            FL              Miami   \n",
       "4415888                      685            FL             Destin   \n",
       "4415890                      751            FL         Wellington   \n",
       "4415891                      750            FL        Miami Beach   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False             True   \n",
       "1                          0              True         False             True   \n",
       "2                         20              True         False             True   \n",
       "3                         20             False         False             True   \n",
       "4                         35              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "4415886                   40             False         False            False   \n",
       "4415887                    0             False         False            False   \n",
       "4415888                   45             False         False            False   \n",
       "4415890                   10             False         False            False   \n",
       "4415891                    9             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                        Electric  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "4415886                       Gas  \n",
       "4415887                       Gas  \n",
       "4415888                       Gas  \n",
       "4415890                       Gas  \n",
       "4415891                       Gas  \n",
       "\n",
       "[4218327 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/FL/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/FL-day/2020-dec-day-FL.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>738e8e6953cc4235c6d459106399d1eda0ea2a36</td>\n",
       "      <td>2020-12-27 14:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>663</td>\n",
       "      <td>668</td>\n",
       "      <td>668</td>\n",
       "      <td>FL</td>\n",
       "      <td>Niceville</td>\n",
       "      <td>29</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9c2eaee56dd3dc278aadd1b505f80555e34aabf7</td>\n",
       "      <td>2020-12-22 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>757</td>\n",
       "      <td>757</td>\n",
       "      <td>FL</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f7fda93a084c48fc27fb866d97a035740ab63bb8</td>\n",
       "      <td>2020-12-23 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>739</td>\n",
       "      <td>739</td>\n",
       "      <td>FL</td>\n",
       "      <td>Altamonte Springs</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>72ead96fa5e75f9860b885ccc10248ff4d4abcd3</td>\n",
       "      <td>2020-12-01 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>665</td>\n",
       "      <td>840</td>\n",
       "      <td>739</td>\n",
       "      <td>FL</td>\n",
       "      <td>Apopka</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>b07d91b813253c89b2d83b35f3778f999ba87915</td>\n",
       "      <td>2020-12-06 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>717</td>\n",
       "      <td>716</td>\n",
       "      <td>656</td>\n",
       "      <td>FL</td>\n",
       "      <td>Miami Springs</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3523556</th>\n",
       "      <td>e1f04e7283ec625fd77870c409b6849ff6a65b73</td>\n",
       "      <td>2020-12-20 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Saint Petersburg</td>\n",
       "      <td>77</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3523557</th>\n",
       "      <td>30cde2844d0dec8628b457fb1de79230a8f0effa</td>\n",
       "      <td>2020-12-15 15:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Boynton Beach</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3523558</th>\n",
       "      <td>42464971f8783480212e38851682da1aac7bb996</td>\n",
       "      <td>2020-12-01 10:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>742</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Naples</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3523559</th>\n",
       "      <td>d6af9ee92932a616a159ebe38d277692fb8130b4</td>\n",
       "      <td>2020-12-12 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>718</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Dunedin</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3523560</th>\n",
       "      <td>2677caa9eda2fcc8b2e21773afbdee6b47ead148</td>\n",
       "      <td>2020-12-16 13:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>FL</td>\n",
       "      <td>Pinecrest</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3374043 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        738e8e6953cc4235c6d459106399d1eda0ea2a36  2020-12-27 14:35:00 UTC   \n",
       "1        9c2eaee56dd3dc278aadd1b505f80555e34aabf7  2020-12-22 17:05:00 UTC   \n",
       "2        f7fda93a084c48fc27fb866d97a035740ab63bb8  2020-12-23 19:00:00 UTC   \n",
       "3        72ead96fa5e75f9860b885ccc10248ff4d4abcd3  2020-12-01 17:00:00 UTC   \n",
       "5        b07d91b813253c89b2d83b35f3778f999ba87915  2020-12-06 18:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "3523556  e1f04e7283ec625fd77870c409b6849ff6a65b73  2020-12-20 17:15:00 UTC   \n",
       "3523557  30cde2844d0dec8628b457fb1de79230a8f0effa  2020-12-15 15:50:00 UTC   \n",
       "3523558  42464971f8783480212e38851682da1aac7bb996  2020-12-01 10:05:00 UTC   \n",
       "3523559  d6af9ee92932a616a159ebe38d277692fb8130b4  2020-12-12 19:05:00 UTC   \n",
       "3523560  2677caa9eda2fcc8b2e21773afbdee6b47ead148  2020-12-16 13:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               663                      668   \n",
       "1           cool          hold               758                      757   \n",
       "2           cool          hold               728                      739   \n",
       "3           cool          auto               665                      840   \n",
       "5           auto          hold               717                      716   \n",
       "...          ...           ...               ...                      ...   \n",
       "3523556     cool          auto               676                      760   \n",
       "3523557     cool          auto               736                      760   \n",
       "3523558     cool          auto               742                      760   \n",
       "3523559     cool          auto               718                      760   \n",
       "3523560     cool          hold               741                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            668            FL          Niceville   \n",
       "1                            757            FL       Oakland Park   \n",
       "2                            739            FL  Altamonte Springs   \n",
       "3                            739            FL             Apopka   \n",
       "5                            656            FL      Miami Springs   \n",
       "...                          ...           ...                ...   \n",
       "3523556                      760            FL   Saint Petersburg   \n",
       "3523557                      760            FL      Boynton Beach   \n",
       "3523558                      760            FL             Naples   \n",
       "3523559                      760            FL            Dunedin   \n",
       "3523560                      760            FL          Pinecrest   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         29             False         False            False   \n",
       "1                         20             False         False            False   \n",
       "2                         15             False         False             True   \n",
       "3                         10              True         False             True   \n",
       "5                         60             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "3523556                   77             False         False             True   \n",
       "3523557                    0             False         False            False   \n",
       "3523558                    9              True         False            False   \n",
       "3523559                   19             False         False             True   \n",
       "3523560                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                        Electric  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "3523556                  Electric  \n",
       "3523557                       Gas  \n",
       "3523558                       Gas  \n",
       "3523559                  Electric  \n",
       "3523560                       Gas  \n",
       "\n",
       "[3374043 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/FL/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/FL/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>713.775000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000901820a1c537e3432a15731cf77e975629e0e</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>727.890756</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>700.000000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>774.085502</td>\n",
       "      <td>799.877323</td>\n",
       "      <td>780.029740</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>747.778309</td>\n",
       "      <td>796.191786</td>\n",
       "      <td>789.730064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>621.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>602.000000</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6800</th>\n",
       "      <td>ffa4468a31a8e35dd633f0022ea0b855501c5ba9</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>718.169048</td>\n",
       "      <td>719.428571</td>\n",
       "      <td>719.392857</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6801</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>736.558282</td>\n",
       "      <td>747.098160</td>\n",
       "      <td>745.914110</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6802</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>728.545455</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>710.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6803</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>738.437500</td>\n",
       "      <td>775.010417</td>\n",
       "      <td>713.031250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6804</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>745.125000</td>\n",
       "      <td>740.312500</td>\n",
       "      <td>739.520833</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25998 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00020f16d85e6d8c70fc67725f03ab06b116fead   dec  2017     auto   \n",
       "1     000901820a1c537e3432a15731cf77e975629e0e   dec  2017     auto   \n",
       "2     0024f0f900a4ff6923e95a2a20ee94c699ecf167   dec  2017     cool   \n",
       "3     0024f0f900a4ff6923e95a2a20ee94c699ecf167   dec  2017     cool   \n",
       "4     0055f438d1ed6195816124e1814d00fe22894e77   dec  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "6800  ffa4468a31a8e35dd633f0022ea0b855501c5ba9   dec  2020     heat   \n",
       "6801  fffcd8322781a8483039a10c259725e9da9cfdec   dec  2020     cool   \n",
       "6802  ffff46e404a9831c28d8be1a3e427a4377690c6b   dec  2020     auto   \n",
       "6803  ffff46e404a9831c28d8be1a3e427a4377690c6b   dec  2020     auto   \n",
       "6804  ffff46e404a9831c28d8be1a3e427a4377690c6b   dec  2020     cool   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             auto  Jacksonville            713.775000   \n",
       "1             auto         Tampa            727.890756   \n",
       "2             auto         Miami            774.085502   \n",
       "3             hold         Miami            747.778309   \n",
       "4             hold         Tampa            621.000000   \n",
       "...            ...           ...                   ...   \n",
       "6800          hold    Palm Coast            718.169048   \n",
       "6801          hold   North Miami            736.558282   \n",
       "6802          auto       Parrish            728.545455   \n",
       "6803          hold       Parrish            738.437500   \n",
       "6804          hold       Parrish            745.125000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      760.000000                   660.000000   \n",
       "1                      750.000000                   700.000000   \n",
       "2                      799.877323                   780.029740   \n",
       "3                      796.191786                   789.730064   \n",
       "4                      750.000000                   602.000000   \n",
       "...                           ...                          ...   \n",
       "6800                   719.428571                   719.392857   \n",
       "6801                   747.098160                   745.914110   \n",
       "6802                   760.000000                   710.000000   \n",
       "6803                   775.010417                   713.031250   \n",
       "6804                   740.312500                   739.520833   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     5.0             False         False             True  \n",
       "1                    17.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     0.0             False         False            False  \n",
       "4                    25.0             False         False             True  \n",
       "...                   ...               ...           ...              ...  \n",
       "6800                 10.0             False         False             True  \n",
       "6801                 70.0             False         False            False  \n",
       "6802                  0.0             False         False             True  \n",
       "6803                  0.0             False         False             True  \n",
       "6804                  0.0             False         False             True  \n",
       "\n",
       "[25998 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/FL/dec/\" + file)\n",
    "    FL_dec = pd.concat([FL_dec, df])\n",
    "    \n",
    "FL_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_dec.to_csv(\"Scraper_Output/State_Month_Day/FL/FL_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/FL/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00020f16d85e6d8c70fc67725f03ab06b116fead</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Jacksonville</td>\n",
       "      <td>782.218335</td>\n",
       "      <td>780.000000</td>\n",
       "      <td>702.132690</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Miami</td>\n",
       "      <td>816.357640</td>\n",
       "      <td>817.450225</td>\n",
       "      <td>784.571897</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0024f0f900a4ff6923e95a2a20ee94c699ecf167</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Miami</td>\n",
       "      <td>802.366295</td>\n",
       "      <td>803.643454</td>\n",
       "      <td>790.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>735.250000</td>\n",
       "      <td>734.833333</td>\n",
       "      <td>660.583333</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0055f438d1ed6195816124e1814d00fe22894e77</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Tampa</td>\n",
       "      <td>750.834862</td>\n",
       "      <td>742.440367</td>\n",
       "      <td>741.862385</td>\n",
       "      <td>25.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21160</th>\n",
       "      <td>ff5c81c4eacfb17ff63cecf31c131332abc79ddc</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Fort Myers</td>\n",
       "      <td>743.019139</td>\n",
       "      <td>739.555024</td>\n",
       "      <td>652.000000</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21161</th>\n",
       "      <td>ffa4468a31a8e35dd633f0022ea0b855501c5ba9</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Palm Coast</td>\n",
       "      <td>760.596356</td>\n",
       "      <td>761.847160</td>\n",
       "      <td>758.819721</td>\n",
       "      <td>10.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21162</th>\n",
       "      <td>ffcf942aa8736eaa25440e6817dea4d9efc3c67f</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oakland Park</td>\n",
       "      <td>737.767442</td>\n",
       "      <td>730.038760</td>\n",
       "      <td>729.782946</td>\n",
       "      <td>30.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21163</th>\n",
       "      <td>fffcd8322781a8483039a10c259725e9da9cfdec</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>North Miami</td>\n",
       "      <td>728.397478</td>\n",
       "      <td>724.251096</td>\n",
       "      <td>723.402412</td>\n",
       "      <td>70.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21164</th>\n",
       "      <td>ffff46e404a9831c28d8be1a3e427a4377690c6b</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Parrish</td>\n",
       "      <td>741.016129</td>\n",
       "      <td>737.701613</td>\n",
       "      <td>737.403226</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135920 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     Identifier Month  Year HvacMode  \\\n",
       "0      00020f16d85e6d8c70fc67725f03ab06b116fead   aug  2017     auto   \n",
       "1      0024f0f900a4ff6923e95a2a20ee94c699ecf167   aug  2017     cool   \n",
       "2      0024f0f900a4ff6923e95a2a20ee94c699ecf167   aug  2017     cool   \n",
       "3      0055f438d1ed6195816124e1814d00fe22894e77   aug  2017     cool   \n",
       "4      0055f438d1ed6195816124e1814d00fe22894e77   aug  2017     cool   \n",
       "...                                         ...   ...   ...      ...   \n",
       "21160  ff5c81c4eacfb17ff63cecf31c131332abc79ddc   jun  2021     cool   \n",
       "21161  ffa4468a31a8e35dd633f0022ea0b855501c5ba9   jun  2021     cool   \n",
       "21162  ffcf942aa8736eaa25440e6817dea4d9efc3c67f   jun  2021     cool   \n",
       "21163  fffcd8322781a8483039a10c259725e9da9cfdec   jun  2021     cool   \n",
       "21164  ffff46e404a9831c28d8be1a3e427a4377690c6b   jun  2021     cool   \n",
       "\n",
       "      CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0              auto  Jacksonville            782.218335   \n",
       "1              auto         Miami            816.357640   \n",
       "2              hold         Miami            802.366295   \n",
       "3              auto         Tampa            735.250000   \n",
       "4              hold         Tampa            750.834862   \n",
       "...             ...           ...                   ...   \n",
       "21160          hold    Fort Myers            743.019139   \n",
       "21161          hold    Palm Coast            760.596356   \n",
       "21162          hold  Oakland Park            737.767442   \n",
       "21163          hold   North Miami            728.397478   \n",
       "21164          hold       Parrish            741.016129   \n",
       "\n",
       "       TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                       780.000000                   702.132690   \n",
       "1                       817.450225                   784.571897   \n",
       "2                       803.643454                   790.000000   \n",
       "3                       734.833333                   660.583333   \n",
       "4                       742.440367                   741.862385   \n",
       "...                            ...                          ...   \n",
       "21160                   739.555024                   652.000000   \n",
       "21161                   761.847160                   758.819721   \n",
       "21162                   730.038760                   729.782946   \n",
       "21163                   724.251096                   723.402412   \n",
       "21164                   737.701613                   737.403226   \n",
       "\n",
       "       Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                      5.0             False         False             True  \n",
       "1                      0.0             False         False            False  \n",
       "2                      0.0             False         False            False  \n",
       "3                     25.0             False         False             True  \n",
       "4                     25.0             False         False             True  \n",
       "...                    ...               ...           ...              ...  \n",
       "21160                 10.0             False         False            False  \n",
       "21161                 10.0             False         False             True  \n",
       "21162                 30.0             False         False            False  \n",
       "21163                 70.0             False         False            False  \n",
       "21164                  0.0             False         False             True  \n",
       "\n",
       "[135920 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "FL_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/FL/\" + file)\n",
    "    FL_all = pd.concat([FL_all, df])\n",
    "    \n",
    "FL_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "FL_all.to_csv(\"Scraper_Output/State_Month_Day/FL_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['FL']\n",
      "Unique jan_2018: ['FL']\n",
      "Unique jan_2019: ['FL']\n",
      "Unique jan_2020: ['FL']\n",
      "Unique jan_2021: ['FL']\n",
      "Unique feb_2017: ['FL']\n",
      "Unique feb_2018: ['FL']\n",
      "Unique feb_2019: ['FL']\n",
      "Unique feb_2020: ['FL']\n",
      "Unique feb_2021: ['FL']\n",
      "Unique jun_2017: ['FL']\n",
      "Unique jun_2018: ['FL']\n",
      "Unique jun_2019: ['FL']\n",
      "Unique jun_2020: ['FL']\n",
      "Unique jun_2021: ['FL']\n",
      "Unique jul_2017: ['FL']\n",
      "Unique jul_2018: ['FL']\n",
      "Unique jul_2019: ['FL']\n",
      "Unique jul_2020: ['FL']\n",
      "Unique jul_2021: ['FL']\n",
      "Unique aug_2017: ['FL']\n",
      "Unique aug_2018: ['FL']\n",
      "Unique aug_2019: ['FL']\n",
      "Unique aug_2020: ['FL']\n",
      "Unique dec_2017: ['FL']\n",
      "Unique dec_2018: ['FL']\n",
      "Unique dec_2019: ['FL']\n",
      "Unique dec_2020: ['FL']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mFLe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}