DYD_NJ-day.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DYD Thermostat Data \n",
    "\n",
    "## Preprocess\n",
    "\n",
    "1. Generated CSV file from queries in BigQueary\n",
    "\n",
    "2. Data separated into states\n",
    "\n",
    "3. Aggregated the data in Pandas by month\n",
    "\n",
    "4. Combine 4 years\n",
    "\n",
    "5. Group by Identifier\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## January"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2017 = pd.read_csv(\"../data_large/NJ-day/2017-jan-day-NJ.csv\")\n",
    "\n",
    "# jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fe013028a9185411235f2cb7ddcb7fdb687017f1</td>\n",
       "      <td>2017-01-18 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Red Bank</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>330477b47f383a9b36a1a7906d462aec91634236</td>\n",
       "      <td>2017-01-19 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>679</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Glassboro</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>955939ffe1d9fd44a65a91f88cf3416591100238</td>\n",
       "      <td>2017-01-25 15:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>710</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Rutherford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e926e5589e6c8e150b37580a1570b0512ce041e3</td>\n",
       "      <td>2017-01-28 19:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Berkeley Heights</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>abffbb0b23f18ab064c8390da5fbb932ccb6b293</td>\n",
       "      <td>2017-01-18 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>641</td>\n",
       "      <td>820</td>\n",
       "      <td>620</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Budd Lake</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401498</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-01-14 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>623</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401499</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-01-01 16:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>678</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401500</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-01-31 13:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>650</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401501</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-01-02 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401502</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-01-04 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>670</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>375501 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       fe013028a9185411235f2cb7ddcb7fdb687017f1  2017-01-18 18:50:00 UTC   \n",
       "1       330477b47f383a9b36a1a7906d462aec91634236  2017-01-19 19:30:00 UTC   \n",
       "2       955939ffe1d9fd44a65a91f88cf3416591100238  2017-01-25 15:35:00 UTC   \n",
       "3       e926e5589e6c8e150b37580a1570b0512ce041e3  2017-01-28 19:40:00 UTC   \n",
       "5       abffbb0b23f18ab064c8390da5fbb932ccb6b293  2017-01-18 16:00:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "401498  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-01-14 12:05:00 UTC   \n",
       "401499  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-01-01 16:00:00 UTC   \n",
       "401500  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-01-31 13:40:00 UTC   \n",
       "401501  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-01-02 13:35:00 UTC   \n",
       "401502  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-01-04 19:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               685                      690   \n",
       "1          heat          hold               679                      670   \n",
       "2          heat          auto               712                      710   \n",
       "3          heat          hold               740                      740   \n",
       "5          heat          auto               641                      820   \n",
       "...         ...           ...               ...                      ...   \n",
       "401498     heat          hold               623                      650   \n",
       "401499     heat          hold               678                      670   \n",
       "401500     heat          hold               645                      650   \n",
       "401501     heat          hold               670                      650   \n",
       "401502     heat          hold               688                      670   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                           690            NJ          Red Bank   \n",
       "1                           670            NJ         Glassboro   \n",
       "2                           710            NJ        Rutherford   \n",
       "3                           740            NJ  Berkeley Heights   \n",
       "5                           620            NJ         Budd Lake   \n",
       "...                         ...           ...               ...   \n",
       "401498                      630            NJ         Englewood   \n",
       "401499                      670            NJ         Englewood   \n",
       "401500                      650            NJ         Englewood   \n",
       "401501                      640            NJ         Englewood   \n",
       "401502                      670            NJ         Englewood   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                       100             False         False            False   \n",
       "1                        55             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "5                        50             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "401498                    0             False         False            False   \n",
       "401499                    0             False         False            False   \n",
       "401500                    0             False         False            False   \n",
       "401501                    0             False         False            False   \n",
       "401502                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "401498                       Gas  \n",
       "401499                       Gas  \n",
       "401500                       Gas  \n",
       "401501                       Gas  \n",
       "401502                       Gas  \n",
       "\n",
       "[375501 rows x 14 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2017.drop(jan_2017[jan_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2017[\"Year\"] = \"2017\"\n",
    "jan_2017[\"Month\"] = \"Jan\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2017 = jan_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00cfe51c25b825270bc4509dfadca8d4f50a4e99</th>\n",
       "      <th>Jan</th>\n",
       "      <th>2017</th>\n",
       "      <th>heat</th>\n",
       "      <th>hold</th>\n",
       "      <th>Medford</th>\n",
       "      <td>689.656934</td>\n",
       "      <td>698.226277</td>\n",
       "      <td>697.970803</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">01344d14a623a52997206e073b17517a488ddb6c</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Piscataway</th>\n",
       "      <td>696.955056</td>\n",
       "      <td>682.685393</td>\n",
       "      <td>682.393258</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Piscataway</th>\n",
       "      <td>702.070866</td>\n",
       "      <td>705.047244</td>\n",
       "      <td>705.047244</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0218f5e261862c4e29bc80b3fd87638a4dd80584</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">auto</th>\n",
       "      <th>auto</th>\n",
       "      <th>Lindenwold</th>\n",
       "      <td>689.480000</td>\n",
       "      <td>751.224615</td>\n",
       "      <td>694.043077</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Lindenwold</th>\n",
       "      <td>702.494949</td>\n",
       "      <td>748.262626</td>\n",
       "      <td>711.262626</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">fe013028a9185411235f2cb7ddcb7fdb687017f1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">2017</th>\n",
       "      <th>auto</th>\n",
       "      <th>hold</th>\n",
       "      <th>Red Bank</th>\n",
       "      <td>659.333333</td>\n",
       "      <td>661.851852</td>\n",
       "      <td>616.111111</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Red Bank</th>\n",
       "      <td>676.682171</td>\n",
       "      <td>689.847545</td>\n",
       "      <td>681.832041</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Red Bank</th>\n",
       "      <td>677.893921</td>\n",
       "      <td>683.466031</td>\n",
       "      <td>682.918951</td>\n",
       "      <td>100.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">fed532fe4fb9b0318ad6822647e83bf9f647877e</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">Jan</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">2017</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">heat</th>\n",
       "      <th>auto</th>\n",
       "      <th>Jersey City</th>\n",
       "      <td>761.935897</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>760.000000</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hold</th>\n",
       "      <th>Jersey City</th>\n",
       "      <td>760.532520</td>\n",
       "      <td>760.004065</td>\n",
       "      <td>759.995935</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>714 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                        Temperature_ctrl_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                689.656934   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway             696.955056   \n",
       "                                                             hold          Piscataway             702.070866   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold             689.480000   \n",
       "                                                             hold          Lindenwold             702.494949   \n",
       "...                                                                                                      ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank               659.333333   \n",
       "                                                    heat     auto          Red Bank               676.682171   \n",
       "                                                             hold          Red Bank               677.893921   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City            761.935897   \n",
       "                                                             hold          Jersey City            760.532520   \n",
       "\n",
       "                                                                                        TemperatureExpectedCool_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                       698.226277   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway                    682.685393   \n",
       "                                                             hold          Piscataway                    705.047244   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold                    751.224615   \n",
       "                                                             hold          Lindenwold                    748.262626   \n",
       "...                                                                                                             ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank                      661.851852   \n",
       "                                                    heat     auto          Red Bank                      689.847545   \n",
       "                                                             hold          Red Bank                      683.466031   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City                   760.000000   \n",
       "                                                             hold          Jersey City                   760.004065   \n",
       "\n",
       "                                                                                        TemperatureExpectedHeat_ave  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                                       \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                       697.970803   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway                    682.393258   \n",
       "                                                             hold          Piscataway                    705.047244   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold                    694.043077   \n",
       "                                                             hold          Lindenwold                    711.262626   \n",
       "...                                                                                                             ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank                      616.111111   \n",
       "                                                    heat     auto          Red Bank                      681.832041   \n",
       "                                                             hold          Red Bank                      682.918951   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City                   760.000000   \n",
       "                                                             hold          Jersey City                   759.995935   \n",
       "\n",
       "                                                                                        Age_of_Home__years_  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                               \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                      0.0   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway                   5.0   \n",
       "                                                             hold          Piscataway                   5.0   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold                  10.0   \n",
       "                                                             hold          Lindenwold                  10.0   \n",
       "...                                                                                                     ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank                   100.0   \n",
       "                                                    heat     auto          Red Bank                   100.0   \n",
       "                                                             hold          Red Bank                   100.0   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City                120.0   \n",
       "                                                             hold          Jersey City                120.0   \n",
       "\n",
       "                                                                                        allowCompWithAux  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                            \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                 False   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway              False   \n",
       "                                                             hold          Piscataway              False   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold               True   \n",
       "                                                             hold          Lindenwold               True   \n",
       "...                                                                                                  ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank                False   \n",
       "                                                    heat     auto          Red Bank                False   \n",
       "                                                             hold          Red Bank                False   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City             False   \n",
       "                                                             hold          Jersey City             False   \n",
       "\n",
       "                                                                                        Has_Electric  \\\n",
       "Identifier                               Month Year HvacMode CalendarEvent City                        \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford             False   \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway          False   \n",
       "                                                             hold          Piscataway          False   \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold          False   \n",
       "                                                             hold          Lindenwold          False   \n",
       "...                                                                                              ...   \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank            False   \n",
       "                                                    heat     auto          Red Bank            False   \n",
       "                                                             hold          Red Bank            False   \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City         False   \n",
       "                                                             hold          Jersey City         False   \n",
       "\n",
       "                                                                                        Has_a_Heat_Pump  \n",
       "Identifier                               Month Year HvacMode CalendarEvent City                          \n",
       "00cfe51c25b825270bc4509dfadca8d4f50a4e99 Jan   2017 heat     hold          Medford                False  \n",
       "01344d14a623a52997206e073b17517a488ddb6c Jan   2017 heat     auto          Piscataway             False  \n",
       "                                                             hold          Piscataway             False  \n",
       "0218f5e261862c4e29bc80b3fd87638a4dd80584 Jan   2017 auto     auto          Lindenwold             False  \n",
       "                                                             hold          Lindenwold             False  \n",
       "...                                                                                                 ...  \n",
       "fe013028a9185411235f2cb7ddcb7fdb687017f1 Jan   2017 auto     hold          Red Bank               False  \n",
       "                                                    heat     auto          Red Bank               False  \n",
       "                                                             hold          Red Bank               False  \n",
       "fed532fe4fb9b0318ad6822647e83bf9f647877e Jan   2017 heat     auto          Jersey City            False  \n",
       "                                                             hold          Jersey City            False  \n",
       "\n",
       "[714 rows x 7 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jan_2017_ave = jan_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "jan_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2017_ave.to_csv(\"data/day/NJ/jan/jan_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2018 = pd.read_csv(\"../data_large/NJ-day/2018-jan-day-NJ.csv\")\n",
    "\n",
    "# jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>577d8d174977a3125a5f92ae94a22b2aaa79e5c7</td>\n",
       "      <td>2018-01-05 16:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Toms River Township</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d37109679aedec1fce5a81b6cc4ed9a7278e7c05</td>\n",
       "      <td>2018-01-24 18:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b597ef7a2b7435e18f0c833e161bad40e65d4085</td>\n",
       "      <td>2018-01-19 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Tinton Falls</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f0376c6b07664a352507d18d771b3afbe349aa77</td>\n",
       "      <td>2018-01-30 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fair Lawn</td>\n",
       "      <td>65</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f07d67d15c361d501977b6d4e5500f2c321f2416</td>\n",
       "      <td>2018-01-27 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>723</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Monmouth Junction</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036656</th>\n",
       "      <td>e712adec63b5ee578e75a6a293bfb048754d7515</td>\n",
       "      <td>2018-01-12 12:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Saddle Brook</td>\n",
       "      <td>75</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036657</th>\n",
       "      <td>057331f5a151522c563bcf0efb49f0b37b733a7d</td>\n",
       "      <td>2018-01-13 11:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Point Pleasant Beach</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036658</th>\n",
       "      <td>3feed3251c993ad584ca33481c115d25de9bad20</td>\n",
       "      <td>2018-01-17 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Union City</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036659</th>\n",
       "      <td>275f12a74bb0b37af045dbd8b1b53d74c9f07946</td>\n",
       "      <td>2018-01-07 15:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Evesham</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036660</th>\n",
       "      <td>3c12bb1b3678c0828a2983a73abf4bc0ae01bdfb</td>\n",
       "      <td>2018-01-18 13:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>690</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Pompton plains</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>986198 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        577d8d174977a3125a5f92ae94a22b2aaa79e5c7  2018-01-05 16:40:00 UTC   \n",
       "1        d37109679aedec1fce5a81b6cc4ed9a7278e7c05  2018-01-24 18:15:00 UTC   \n",
       "2        b597ef7a2b7435e18f0c833e161bad40e65d4085  2018-01-19 13:20:00 UTC   \n",
       "3        f0376c6b07664a352507d18d771b3afbe349aa77  2018-01-30 17:25:00 UTC   \n",
       "4        f07d67d15c361d501977b6d4e5500f2c321f2416  2018-01-27 17:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1036656  e712adec63b5ee578e75a6a293bfb048754d7515  2018-01-12 12:05:00 UTC   \n",
       "1036657  057331f5a151522c563bcf0efb49f0b37b733a7d  2018-01-13 11:30:00 UTC   \n",
       "1036658  3feed3251c993ad584ca33481c115d25de9bad20  2018-01-17 19:20:00 UTC   \n",
       "1036659  275f12a74bb0b37af045dbd8b1b53d74c9f07946  2018-01-07 15:40:00 UTC   \n",
       "1036660  3c12bb1b3678c0828a2983a73abf4bc0ae01bdfb  2018-01-18 13:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               712                      705   \n",
       "1           heat          hold               732                      728   \n",
       "2           heat          hold               681                      675   \n",
       "3           heat          hold               702                      705   \n",
       "4           auto          hold               723                      840   \n",
       "...          ...           ...               ...                      ...   \n",
       "1036656     auto          auto               699                      760   \n",
       "1036657     auto          auto               700                      760   \n",
       "1036658     heat          auto               741                      760   \n",
       "1036659     heat          auto               753                      760   \n",
       "1036660     heat          auto               690                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                  City  \\\n",
       "0                            705            NJ   Toms River Township   \n",
       "1                            728            NJ               Jackson   \n",
       "2                            675            NJ          Tinton Falls   \n",
       "3                            705            NJ             Fair Lawn   \n",
       "4                            690            NJ     Monmouth Junction   \n",
       "...                          ...           ...                   ...   \n",
       "1036656                      700            NJ          Saddle Brook   \n",
       "1036657                      700            NJ  Point Pleasant Beach   \n",
       "1036658                      760            NJ            Union City   \n",
       "1036659                      760            NJ               Evesham   \n",
       "1036660                      690            NJ       Pompton plains    \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False            False   \n",
       "1                         30             False         False            False   \n",
       "2                         65             False         False            False   \n",
       "3                         65             False         False            False   \n",
       "4                         17             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1036656                   75             False         False            False   \n",
       "1036657                   45             False         False            False   \n",
       "1036658                   90              True         False            False   \n",
       "1036659                   16             False         False            False   \n",
       "1036660                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1036656                       Gas  \n",
       "1036657                       Gas  \n",
       "1036658                       Gas  \n",
       "1036659                       Gas  \n",
       "1036660                       Gas  \n",
       "\n",
       "[986198 rows x 14 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2018.drop(jan_2018[jan_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2018[\"Year\"] = \"2018\"\n",
    "jan_2018[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2018 = jan_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2018_ave = jan_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2018_ave.to_csv(\"data/day/NJ/jan/jan_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2019 = pd.read_csv(\"../data_large/NJ-day/2019-jan-day-NJ.csv\")\n",
    "\n",
    "# jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a1348dbfed8425aee385aca041820e0deadadb82</td>\n",
       "      <td>2019-01-03 11:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>626</td>\n",
       "      <td>626</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5f0e2feee4d0ed9cccba0c60f7b3be843059bb4e</td>\n",
       "      <td>2019-01-20 18:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>693</td>\n",
       "      <td>712</td>\n",
       "      <td>712</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Sea Girt</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2eec69a6ea2000df3d4a08508e75b2691b0fb900</td>\n",
       "      <td>2019-01-26 18:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>745</td>\n",
       "      <td>645</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montville</td>\n",
       "      <td>48</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>61d5b67391246e1c46d918cff478154ebf4ae71b</td>\n",
       "      <td>2019-01-22 15:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>795</td>\n",
       "      <td>715</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Green Brook</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>138f670a27f442a2620981e94b5b73e619629984</td>\n",
       "      <td>2019-01-31 18:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>NJ</td>\n",
       "      <td>toms river</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1622676</th>\n",
       "      <td>754a29221076050e925e6d09922eb1237a53e3db</td>\n",
       "      <td>2019-01-21 18:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>647</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Woodstown</td>\n",
       "      <td>85</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1622677</th>\n",
       "      <td>4cb39ac043209d58e79a06402eac00eecc6adbee</td>\n",
       "      <td>2019-01-27 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Wayne</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1622678</th>\n",
       "      <td>cec5effeb362e573c7488f6f5445d3f6f2339aea</td>\n",
       "      <td>2019-01-03 13:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montclair</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1622679</th>\n",
       "      <td>a027131b5db9cc12c3c5e6117ab7d412ceddaa62</td>\n",
       "      <td>2019-01-12 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Warren</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1622680</th>\n",
       "      <td>ba750e1e8ea0125697626086bdd9a980b401746a</td>\n",
       "      <td>2019-01-09 13:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1536669 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        a1348dbfed8425aee385aca041820e0deadadb82  2019-01-03 11:30:00 UTC   \n",
       "1        5f0e2feee4d0ed9cccba0c60f7b3be843059bb4e  2019-01-20 18:55:00 UTC   \n",
       "2        2eec69a6ea2000df3d4a08508e75b2691b0fb900  2019-01-26 18:40:00 UTC   \n",
       "3        61d5b67391246e1c46d918cff478154ebf4ae71b  2019-01-22 15:55:00 UTC   \n",
       "4        138f670a27f442a2620981e94b5b73e619629984  2019-01-31 18:50:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1622676  754a29221076050e925e6d09922eb1237a53e3db  2019-01-21 18:00:00 UTC   \n",
       "1622677  4cb39ac043209d58e79a06402eac00eecc6adbee  2019-01-27 16:15:00 UTC   \n",
       "1622678  cec5effeb362e573c7488f6f5445d3f6f2339aea  2019-01-03 13:30:00 UTC   \n",
       "1622679  a027131b5db9cc12c3c5e6117ab7d412ceddaa62  2019-01-12 17:05:00 UTC   \n",
       "1622680  ba750e1e8ea0125697626086bdd9a980b401746a  2019-01-09 13:05:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               698                      626   \n",
       "1           heat          hold               693                      712   \n",
       "2           auto          hold               645                      745   \n",
       "3           auto          hold               708                      795   \n",
       "4           heat          hold               729                      734   \n",
       "...          ...           ...               ...                      ...   \n",
       "1622676     auto          auto               647                      760   \n",
       "1622677     auto          hold               706                      760   \n",
       "1622678     auto          hold               676                      760   \n",
       "1622679     auto          hold               701                      760   \n",
       "1622680     heat          hold               759                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                            626            NJ    Ridgewood   \n",
       "1                            712            NJ     Sea Girt   \n",
       "2                            645            NJ    Montville   \n",
       "3                            715            NJ  Green Brook   \n",
       "4                            734            NJ   toms river   \n",
       "...                          ...           ...          ...   \n",
       "1622676                      650            NJ    Woodstown   \n",
       "1622677                      710            NJ        Wayne   \n",
       "1622678                      680            NJ    Montclair   \n",
       "1622679                      700            NJ       Warren   \n",
       "1622680                      760            NJ       Berlin   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         60             False         False            False   \n",
       "1                          5             False         False            False   \n",
       "2                         48             False         False            False   \n",
       "3                         30             False         False            False   \n",
       "4                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1622676                   85             False         False            False   \n",
       "1622677                   25             False         False            False   \n",
       "1622678                    0             False         False            False   \n",
       "1622679                   17             False         False            False   \n",
       "1622680                   17             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1622676                       Gas  \n",
       "1622677                       Gas  \n",
       "1622678                       Gas  \n",
       "1622679                       Gas  \n",
       "1622680                       Gas  \n",
       "\n",
       "[1536669 rows x 14 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2019.drop(jan_2019[jan_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2019[\"Year\"] = \"2019\"\n",
    "jan_2019[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2019 = jan_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2019_ave = jan_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2019_ave.to_csv(\"data/day/NJ/jan/jan_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2020 = pd.read_csv(\"../data_large/NJ-day/2020-jan-day-NJ.csv\")\n",
    "\n",
    "# jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>975e208dd09c056c95037ce654a74fdb8d8caf4d</td>\n",
       "      <td>2020-01-21 14:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>674</td>\n",
       "      <td>630</td>\n",
       "      <td>630</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Freehold</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fe4927c7c7792ea973b4840bbbcf76757cc61fe6</td>\n",
       "      <td>2020-01-14 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>745</td>\n",
       "      <td>695</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Princeton</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2b10c358c75983a3bd811c5645e19520c22cfbe5</td>\n",
       "      <td>2020-01-29 13:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mahwah</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4dacf6559c4e2555cf463516914138ef093acab5</td>\n",
       "      <td>2020-01-08 13:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>662</td>\n",
       "      <td>672</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ringwood</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c74773486a56f2b0f50c7d3e78e3e22b3a05ba53</td>\n",
       "      <td>2020-01-27 13:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>762</td>\n",
       "      <td>712</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Bridgewater</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860198</th>\n",
       "      <td>1e01e0692664bb154774669a1330cd15ecccb453</td>\n",
       "      <td>2020-01-12 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Franklin Township</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860199</th>\n",
       "      <td>2406f1c08f3758a93961d01674e31daf625fab21</td>\n",
       "      <td>2020-01-04 12:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>760</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Madison</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860200</th>\n",
       "      <td>d07c7da92a8a543a676bf4c3cf458f11ca8dba9e</td>\n",
       "      <td>2020-01-20 19:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>748</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fort Lee</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860202</th>\n",
       "      <td>0885a196a9c6a38baf5f352bbeefa6e75b5cf40c</td>\n",
       "      <td>2020-01-04 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Point Pleasant Boro</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1860203</th>\n",
       "      <td>d07c7da92a8a543a676bf4c3cf458f11ca8dba9e</td>\n",
       "      <td>2020-01-27 11:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fort Lee</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1744584 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        975e208dd09c056c95037ce654a74fdb8d8caf4d  2020-01-21 14:15:00 UTC   \n",
       "1        fe4927c7c7792ea973b4840bbbcf76757cc61fe6  2020-01-14 13:15:00 UTC   \n",
       "2        2b10c358c75983a3bd811c5645e19520c22cfbe5  2020-01-29 13:15:00 UTC   \n",
       "3        4dacf6559c4e2555cf463516914138ef093acab5  2020-01-08 13:20:00 UTC   \n",
       "4        c74773486a56f2b0f50c7d3e78e3e22b3a05ba53  2020-01-27 13:25:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1860198  1e01e0692664bb154774669a1330cd15ecccb453  2020-01-12 12:05:00 UTC   \n",
       "1860199  2406f1c08f3758a93961d01674e31daf625fab21  2020-01-04 12:50:00 UTC   \n",
       "1860200  d07c7da92a8a543a676bf4c3cf458f11ca8dba9e  2020-01-20 19:05:00 UTC   \n",
       "1860202  0885a196a9c6a38baf5f352bbeefa6e75b5cf40c  2020-01-04 19:20:00 UTC   \n",
       "1860203  d07c7da92a8a543a676bf4c3cf458f11ca8dba9e  2020-01-27 11:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          auto               674                      630   \n",
       "1           auto          hold               705                      745   \n",
       "2           heat          hold               716                      718   \n",
       "3           heat          auto               662                      672   \n",
       "4           auto          hold               707                      762   \n",
       "...          ...           ...               ...                      ...   \n",
       "1860198     heat          hold               758                      760   \n",
       "1860199     auto          hold               654                      760   \n",
       "1860200     heat          hold               748                      760   \n",
       "1860202     heat          hold               757                      760   \n",
       "1860203     heat          hold               755                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            630            NJ             Freehold   \n",
       "1                            695            NJ            Princeton   \n",
       "2                            718            NJ               Mahwah   \n",
       "3                            670            NJ             Ringwood   \n",
       "4                            712            NJ          Bridgewater   \n",
       "...                          ...           ...                  ...   \n",
       "1860198                      760            NJ    Franklin Township   \n",
       "1860199                      640            NJ              Madison   \n",
       "1860200                      760            NJ             Fort Lee   \n",
       "1860202                      760            NJ  Point Pleasant Boro   \n",
       "1860203                      760            NJ             Fort Lee   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         25             False         False            False   \n",
       "1                          9              True         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                         30              True         False            False   \n",
       "4                         20              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1860198                    0              True         False            False   \n",
       "1860199                   90             False         False            False   \n",
       "1860200                   99             False         False            False   \n",
       "1860202                    0              True         False            False   \n",
       "1860203                   99             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1860198                       Gas  \n",
       "1860199                       Gas  \n",
       "1860200                       Gas  \n",
       "1860202                       Gas  \n",
       "1860203                       Gas  \n",
       "\n",
       "[1744584 rows x 14 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2020.drop(jan_2020[jan_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2020[\"Year\"] = \"2020\"\n",
    "jan_2020[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2020 = jan_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2020_ave = jan_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2020_ave.to_csv(\"data/day/NJ/jan/jan_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 January Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jan_2021 = pd.read_csv(\"../data_large/NJ-day/2021-jan-day-NJ.csv\")\n",
    "\n",
    "# jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6d9ae47183e5b4373e62c07c96f2abba847d3694</td>\n",
       "      <td>2021-01-10 17:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>709</td>\n",
       "      <td>709</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Chesterfield</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9a06657297ff3a5ae7889f50724c76bd6ab0ba1b</td>\n",
       "      <td>2021-01-18 10:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>711</td>\n",
       "      <td>711</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3734528620473667d6bb2e3b5e01bddb55a5a781</td>\n",
       "      <td>2021-01-31 11:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montclair</td>\n",
       "      <td>90</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>76b3c925ff149d91c908ee800517742bc3118821</td>\n",
       "      <td>2021-01-27 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>673</td>\n",
       "      <td>686</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lindenwold</td>\n",
       "      <td>69</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9a06657297ff3a5ae7889f50724c76bd6ab0ba1b</td>\n",
       "      <td>2021-01-27 19:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089482</th>\n",
       "      <td>d2b4b0edff4d937154c31af3dbc4c54bbccb5179</td>\n",
       "      <td>2021-01-18 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Evesham</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089483</th>\n",
       "      <td>82e64c33a0e15163337ea079b31c43d313ce3f90</td>\n",
       "      <td>2021-01-25 17:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Manahawkin</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089484</th>\n",
       "      <td>0bbc5cd611212f85a36f79f4ab8994ddb2dcea95</td>\n",
       "      <td>2021-01-22 13:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Brunswick</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089485</th>\n",
       "      <td>d07c7da92a8a543a676bf4c3cf458f11ca8dba9e</td>\n",
       "      <td>2021-01-29 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fort Lee</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1089486</th>\n",
       "      <td>bc051604eda6efca83179583489ea71449577a0f</td>\n",
       "      <td>2021-01-10 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Maplewood</td>\n",
       "      <td>110</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1012078 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        6d9ae47183e5b4373e62c07c96f2abba847d3694  2021-01-10 17:40:00 UTC   \n",
       "1        9a06657297ff3a5ae7889f50724c76bd6ab0ba1b  2021-01-18 10:10:00 UTC   \n",
       "2        3734528620473667d6bb2e3b5e01bddb55a5a781  2021-01-31 11:50:00 UTC   \n",
       "3        76b3c925ff149d91c908ee800517742bc3118821  2021-01-27 12:35:00 UTC   \n",
       "4        9a06657297ff3a5ae7889f50724c76bd6ab0ba1b  2021-01-27 19:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1089482  d2b4b0edff4d937154c31af3dbc4c54bbccb5179  2021-01-18 13:50:00 UTC   \n",
       "1089483  82e64c33a0e15163337ea079b31c43d313ce3f90  2021-01-25 17:35:00 UTC   \n",
       "1089484  0bbc5cd611212f85a36f79f4ab8994ddb2dcea95  2021-01-22 13:15:00 UTC   \n",
       "1089485  d07c7da92a8a543a676bf4c3cf458f11ca8dba9e  2021-01-29 14:45:00 UTC   \n",
       "1089486  bc051604eda6efca83179583489ea71449577a0f  2021-01-10 12:35:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               719                      709   \n",
       "1           heat          hold               703                      711   \n",
       "2           heat          hold               727                      729   \n",
       "3           heat          hold               673                      686   \n",
       "4           heat          hold               729                      729   \n",
       "...          ...           ...               ...                      ...   \n",
       "1089482     auto          hold               708                      760   \n",
       "1089483     auto          hold               710                      760   \n",
       "1089484     auto          hold               685                      760   \n",
       "1089485     heat          hold               757                      760   \n",
       "1089486     heat          hold               750                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            709            NJ     Chesterfield   \n",
       "1                            711            NJ       Middletown   \n",
       "2                            729            NJ        Montclair   \n",
       "3                            680            NJ       Lindenwold   \n",
       "4                            729            NJ       Middletown   \n",
       "...                          ...           ...              ...   \n",
       "1089482                      710            NJ          Evesham   \n",
       "1089483                      710            NJ       Manahawkin   \n",
       "1089484                      690            NJ  North Brunswick   \n",
       "1089485                      760            NJ         Fort Lee   \n",
       "1089486                      760            NJ        Maplewood   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         69             False         False            False   \n",
       "2                         90              True         False            False   \n",
       "3                         69              True         False            False   \n",
       "4                         69             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1089482                    9              True         False            False   \n",
       "1089483                    0             False         False            False   \n",
       "1089484                   35             False         False            False   \n",
       "1089485                   99             False         False            False   \n",
       "1089486                  110             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1089482                       Gas  \n",
       "1089483                       Gas  \n",
       "1089484                       Gas  \n",
       "1089485                       Gas  \n",
       "1089486                       Gas  \n",
       "\n",
       "[1012078 rows x 14 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jan_2021.drop(jan_2021[jan_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jan_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jan_2021[\"Year\"] = \"2021\"\n",
    "jan_2021[\"Month\"] = \"Jan\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jan_2021 = jan_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jan_2021_ave = jan_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jan_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jan_2021_ave.to_csv(\"data/day/NJ/jan/jan_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/jan/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00cfe51c25b825270bc4509dfadca8d4f50a4e99</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medford</td>\n",
       "      <td>689.656934</td>\n",
       "      <td>698.226277</td>\n",
       "      <td>697.970803</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01344d14a623a52997206e073b17517a488ddb6c</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Piscataway</td>\n",
       "      <td>696.955056</td>\n",
       "      <td>682.685393</td>\n",
       "      <td>682.393258</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>01344d14a623a52997206e073b17517a488ddb6c</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Piscataway</td>\n",
       "      <td>702.070866</td>\n",
       "      <td>705.047244</td>\n",
       "      <td>705.047244</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0218f5e261862c4e29bc80b3fd87638a4dd80584</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Lindenwold</td>\n",
       "      <td>689.480000</td>\n",
       "      <td>751.224615</td>\n",
       "      <td>694.043077</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0218f5e261862c4e29bc80b3fd87638a4dd80584</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Lindenwold</td>\n",
       "      <td>702.494949</td>\n",
       "      <td>748.262626</td>\n",
       "      <td>711.262626</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>fd608a31545a8a10a6cb5ed7a74d21e831339989</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>683.169317</td>\n",
       "      <td>690.681946</td>\n",
       "      <td>690.564079</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1276</th>\n",
       "      <td>fe66c5051b7e13329c0c52a696a10794fa5efd7d</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Swedesboro</td>\n",
       "      <td>689.569444</td>\n",
       "      <td>690.819444</td>\n",
       "      <td>689.819444</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1277</th>\n",
       "      <td>fe6af6e7fde79582ca0fa187198d1240b25652ee</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Phillipsburg</td>\n",
       "      <td>656.256098</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>660.000000</td>\n",
       "      <td>50.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1278</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>687.153084</td>\n",
       "      <td>693.581498</td>\n",
       "      <td>690.455947</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1279</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>Jan</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>699.811159</td>\n",
       "      <td>727.085837</td>\n",
       "      <td>701.793991</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8304 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     00cfe51c25b825270bc4509dfadca8d4f50a4e99   Jan  2017     heat   \n",
       "1     01344d14a623a52997206e073b17517a488ddb6c   Jan  2017     heat   \n",
       "2     01344d14a623a52997206e073b17517a488ddb6c   Jan  2017     heat   \n",
       "3     0218f5e261862c4e29bc80b3fd87638a4dd80584   Jan  2017     auto   \n",
       "4     0218f5e261862c4e29bc80b3fd87638a4dd80584   Jan  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1275  fd608a31545a8a10a6cb5ed7a74d21e831339989   Jan  2021     heat   \n",
       "1276  fe66c5051b7e13329c0c52a696a10794fa5efd7d   Jan  2021     heat   \n",
       "1277  fe6af6e7fde79582ca0fa187198d1240b25652ee   Jan  2021     heat   \n",
       "1278  ff635b7556b71ecf409661242fa988bf1361c850   Jan  2021     heat   \n",
       "1279  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   Jan  2021     auto   \n",
       "\n",
       "     CalendarEvent          City  Temperature_ctrl_ave  \\\n",
       "0             hold       Medford            689.656934   \n",
       "1             auto    Piscataway            696.955056   \n",
       "2             hold    Piscataway            702.070866   \n",
       "3             auto    Lindenwold            689.480000   \n",
       "4             hold    Lindenwold            702.494949   \n",
       "...            ...           ...                   ...   \n",
       "1275          hold     Westfield            683.169317   \n",
       "1276          hold    Swedesboro            689.569444   \n",
       "1277          hold  Phillipsburg            656.256098   \n",
       "1278          hold        newark            687.153084   \n",
       "1279          hold       Maywood            699.811159   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      698.226277                   697.970803   \n",
       "1                      682.685393                   682.393258   \n",
       "2                      705.047244                   705.047244   \n",
       "3                      751.224615                   694.043077   \n",
       "4                      748.262626                   711.262626   \n",
       "...                           ...                          ...   \n",
       "1275                   690.681946                   690.564079   \n",
       "1276                   690.819444                   689.819444   \n",
       "1277                   660.000000                   660.000000   \n",
       "1278                   693.581498                   690.455947   \n",
       "1279                   727.085837                   701.793991   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     5.0             False         False            False  \n",
       "2                     5.0             False         False            False  \n",
       "3                    10.0              True         False            False  \n",
       "4                    10.0              True         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1275                 50.0             False         False            False  \n",
       "1276                 10.0              True         False            False  \n",
       "1277                 50.0             False         False            False  \n",
       "1278                120.0             False         False            False  \n",
       "1279                 90.0             False         False            False  \n",
       "\n",
       "[8304 rows x 13 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_jan = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/jan/\" + file)\n",
    "    NJ_jan = pd.concat([NJ_jan, df])\n",
    "    \n",
    "NJ_jan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_jan.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_jan.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## February"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2017 = pd.read_csv(\"../data_large/NJ-day/2017-feb-day-NJ.csv\")\n",
    "\n",
    "# feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d3c6c4504a7501861b2f21c71016e8484a441958</td>\n",
       "      <td>2017-02-04 13:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>740</td>\n",
       "      <td>740</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mount Holly</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>63dfcb9581139a2d268e9495d886d661e162b1cd</td>\n",
       "      <td>2017-02-13 16:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>719</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Highlands</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8aea502962b5b263b484c9758ce83a542d441265</td>\n",
       "      <td>2017-02-05 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>664</td>\n",
       "      <td>650</td>\n",
       "      <td>630</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hamilton</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16af22058149fc79833d767883046d2083b8ba64</td>\n",
       "      <td>2017-02-12 09:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>685</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Orange</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d2535263ab073546e3bc0f3c6b92de5eeb960913</td>\n",
       "      <td>2017-02-05 19:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>766</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Rutherford</td>\n",
       "      <td>96</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359472</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-02-14 13:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>675</td>\n",
       "      <td>680</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359473</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-02-09 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>641</td>\n",
       "      <td>650</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359474</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-02-18 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359475</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-02-28 12:35:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>660</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359476</th>\n",
       "      <td>f5c547be91b176e3bb84401fc6658ac7fbc5f937</td>\n",
       "      <td>2017-02-05 11:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>690</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Englewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>335491 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       d3c6c4504a7501861b2f21c71016e8484a441958  2017-02-04 13:45:00 UTC   \n",
       "1       63dfcb9581139a2d268e9495d886d661e162b1cd  2017-02-13 16:25:00 UTC   \n",
       "2       8aea502962b5b263b484c9758ce83a542d441265  2017-02-05 15:15:00 UTC   \n",
       "3       16af22058149fc79833d767883046d2083b8ba64  2017-02-12 09:25:00 UTC   \n",
       "4       d2535263ab073546e3bc0f3c6b92de5eeb960913  2017-02-05 19:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "359472  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-02-14 13:10:00 UTC   \n",
       "359473  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-02-09 16:15:00 UTC   \n",
       "359474  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-02-18 12:20:00 UTC   \n",
       "359475  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-02-28 12:35:00 UTC   \n",
       "359476  f5c547be91b176e3bb84401fc6658ac7fbc5f937  2017-02-05 11:30:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               742                      740   \n",
       "1          heat          auto               719                      720   \n",
       "2          heat          auto               664                      650   \n",
       "3          heat          hold               685                      690   \n",
       "4          heat          auto               766                      680   \n",
       "...         ...           ...               ...                      ...   \n",
       "359472     heat          hold               675                      680   \n",
       "359473     heat          hold               641                      650   \n",
       "359474     heat          hold               714                      720   \n",
       "359475     heat          hold               655                      660   \n",
       "359476     heat          hold               686                      690   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           740            NJ  Mount Holly   \n",
       "1                           720            NJ    Highlands   \n",
       "2                           630            NJ     Hamilton   \n",
       "3                           690            NJ  West Orange   \n",
       "4                           680            NJ   Rutherford   \n",
       "...                         ...           ...          ...   \n",
       "359472                      680            NJ    Englewood   \n",
       "359473                      640            NJ    Englewood   \n",
       "359474                      720            NJ    Englewood   \n",
       "359475                      660            NJ    Englewood   \n",
       "359476                      690            NJ    Englewood   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        50             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                        30             False         False            False   \n",
       "3                        25             False         False            False   \n",
       "4                        96             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "359472                    0             False         False            False   \n",
       "359473                    0             False         False            False   \n",
       "359474                    0             False         False            False   \n",
       "359475                    0             False         False            False   \n",
       "359476                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "359472                       Gas  \n",
       "359473                       Gas  \n",
       "359474                       Gas  \n",
       "359475                       Gas  \n",
       "359476                       Gas  \n",
       "\n",
       "[335491 rows x 14 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2017.drop(feb_2017[feb_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2017[\"Year\"] = \"2017\"\n",
    "feb_2017[\"Month\"] = \"feb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2017 = feb_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2017_ave = feb_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2017_ave.to_csv(\"data/day/NJ/feb/feb_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2018 = pd.read_csv(\"../data_large/NJ-day/2018-feb-day-NJ.csv\")\n",
    "\n",
    "# feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8326662d3d25b26835ae230895324431d31ad774</td>\n",
       "      <td>2018-02-22 16:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>714</td>\n",
       "      <td>708</td>\n",
       "      <td>708</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Leonia</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dda598d15aed73aa5988163c7eda93f005c650b6</td>\n",
       "      <td>2018-02-19 11:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>671</td>\n",
       "      <td>675</td>\n",
       "      <td>675</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Wall</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bad2d0d1a1383feaedc836b267505beee1a40149</td>\n",
       "      <td>2018-02-23 16:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>712</td>\n",
       "      <td>765</td>\n",
       "      <td>715</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Long Branch</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8326662d3d25b26835ae230895324431d31ad774</td>\n",
       "      <td>2018-02-10 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>719</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Leonia</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0735fd45c22def7a595e22134db926c8bc02d5c9</td>\n",
       "      <td>2018-02-03 14:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>689</td>\n",
       "      <td>640</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Milford</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959300</th>\n",
       "      <td>4c7daababb5aa316174d3155f664373420731e32</td>\n",
       "      <td>2018-02-18 18:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>688</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Plumsted</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959301</th>\n",
       "      <td>081b1ac4839127f89386f2192a9862aeca8176b2</td>\n",
       "      <td>2018-02-13 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>622</td>\n",
       "      <td>760</td>\n",
       "      <td>610</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Paramus</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959302</th>\n",
       "      <td>a138cf72954559db33a45516278b0c42b4c95d3c</td>\n",
       "      <td>2018-02-27 13:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Trenton</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959303</th>\n",
       "      <td>193be8b5b1d1c343b24f78aa1adc3f719c7d6484</td>\n",
       "      <td>2018-02-17 14:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>654</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Scotch Plains</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959304</th>\n",
       "      <td>add00bd385c4dea1095432dc6f96b8c271d83184</td>\n",
       "      <td>2018-02-21 13:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>760</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mount Laurel</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>898809 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       8326662d3d25b26835ae230895324431d31ad774  2018-02-22 16:15:00 UTC   \n",
       "1       dda598d15aed73aa5988163c7eda93f005c650b6  2018-02-19 11:50:00 UTC   \n",
       "2       bad2d0d1a1383feaedc836b267505beee1a40149  2018-02-23 16:05:00 UTC   \n",
       "3       8326662d3d25b26835ae230895324431d31ad774  2018-02-10 19:00:00 UTC   \n",
       "4       0735fd45c22def7a595e22134db926c8bc02d5c9  2018-02-03 14:45:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "959300  4c7daababb5aa316174d3155f664373420731e32  2018-02-18 18:25:00 UTC   \n",
       "959301  081b1ac4839127f89386f2192a9862aeca8176b2  2018-02-13 15:40:00 UTC   \n",
       "959302  a138cf72954559db33a45516278b0c42b4c95d3c  2018-02-27 13:00:00 UTC   \n",
       "959303  193be8b5b1d1c343b24f78aa1adc3f719c7d6484  2018-02-17 14:15:00 UTC   \n",
       "959304  add00bd385c4dea1095432dc6f96b8c271d83184  2018-02-21 13:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               714                      708   \n",
       "1          heat          hold               671                      675   \n",
       "2          auto          auto               712                      765   \n",
       "3          heat          hold               719                      719   \n",
       "4          heat          auto               689                      640   \n",
       "...         ...           ...               ...                      ...   \n",
       "959300     auto          auto               688                      760   \n",
       "959301     auto          auto               622                      760   \n",
       "959302     heat          auto               756                      760   \n",
       "959303     auto          hold               654                      760   \n",
       "959304     auto          auto               675                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                           708            NJ         Leonia   \n",
       "1                           675            NJ           Wall   \n",
       "2                           715            NJ    Long Branch   \n",
       "3                           719            NJ         Leonia   \n",
       "4                           640            NJ   West Milford   \n",
       "...                         ...           ...            ...   \n",
       "959300                      670            NJ       Plumsted   \n",
       "959301                      610            NJ        Paramus   \n",
       "959302                      760            NJ        Trenton   \n",
       "959303                      650            NJ  Scotch Plains   \n",
       "959304                      650            NJ   Mount Laurel   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        90             False         False            False   \n",
       "1                         0             False         False            False   \n",
       "2                         0              True         False            False   \n",
       "3                        90             False         False            False   \n",
       "4                        45             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "959300                   20             False         False            False   \n",
       "959301                   20             False         False            False   \n",
       "959302                   30             False         False            False   \n",
       "959303                   70             False         False            False   \n",
       "959304                   37             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "959300                       Gas  \n",
       "959301                       Gas  \n",
       "959302                       Gas  \n",
       "959303                       Gas  \n",
       "959304                       Gas  \n",
       "\n",
       "[898809 rows x 14 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2018.drop(feb_2018[feb_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2018[\"Year\"] = \"2018\"\n",
    "feb_2018[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2018 = feb_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2018_ave = feb_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2018_ave.to_csv(\"data/day/NJ/feb/feb_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2019 = pd.read_csv(\"../data_large/NJ-day/2019-feb-day-NJ.csv\")\n",
    "\n",
    "# feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8f21e67628d86d4d95831e3fa9eb69705f381d8c</td>\n",
       "      <td>2019-02-23 15:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>644</td>\n",
       "      <td>651</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4ec3904e4ecf43d0e7b465e6e1ea6a587e543ee2</td>\n",
       "      <td>2019-02-18 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Sayreville</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>712b2b3bd64c28fda669dd129001570ce44b39af</td>\n",
       "      <td>2019-02-16 17:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>689</td>\n",
       "      <td>658</td>\n",
       "      <td>658</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Elmwood Park</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9cf1156c73de72dc595489619d33f81c3847ea96</td>\n",
       "      <td>2019-02-23 18:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Trenton</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2ae1dca646d7c84098b1c0ce89ae17c0f41c8b0c</td>\n",
       "      <td>2019-02-18 12:30:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>691</td>\n",
       "      <td>840</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mount Laurel</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142197</th>\n",
       "      <td>9bd8656fb70ecba362d749a6f894dc56f82107ee</td>\n",
       "      <td>2019-02-16 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>699</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Wenonah</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142198</th>\n",
       "      <td>d942a8853fab067ec80c18ad5c1c8f59ed79cef7</td>\n",
       "      <td>2019-02-03 12:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Evesham Township</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142199</th>\n",
       "      <td>d01223e33a8e008d33116a5619e808ffee8154e9</td>\n",
       "      <td>2019-02-24 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>707</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Glen Rock</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142200</th>\n",
       "      <td>1af2a6b1c8114d8cca0b140a5b59f06bc81d5899</td>\n",
       "      <td>2019-02-25 12:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1142201</th>\n",
       "      <td>0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea</td>\n",
       "      <td>2019-02-22 11:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Glassboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1066998 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        8f21e67628d86d4d95831e3fa9eb69705f381d8c  2019-02-23 15:30:00 UTC   \n",
       "1        4ec3904e4ecf43d0e7b465e6e1ea6a587e543ee2  2019-02-18 12:05:00 UTC   \n",
       "3        712b2b3bd64c28fda669dd129001570ce44b39af  2019-02-16 17:00:00 UTC   \n",
       "4        9cf1156c73de72dc595489619d33f81c3847ea96  2019-02-23 18:45:00 UTC   \n",
       "5        2ae1dca646d7c84098b1c0ce89ae17c0f41c8b0c  2019-02-18 12:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1142197  9bd8656fb70ecba362d749a6f894dc56f82107ee  2019-02-16 13:55:00 UTC   \n",
       "1142198  d942a8853fab067ec80c18ad5c1c8f59ed79cef7  2019-02-03 12:50:00 UTC   \n",
       "1142199  d01223e33a8e008d33116a5619e808ffee8154e9  2019-02-24 18:30:00 UTC   \n",
       "1142200  1af2a6b1c8114d8cca0b140a5b59f06bc81d5899  2019-02-25 12:30:00 UTC   \n",
       "1142201  0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea  2019-02-22 11:55:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          auto               644                      651   \n",
       "1           heat          hold               705                      705   \n",
       "3           heat          hold               689                      658   \n",
       "4           heat          hold               676                      677   \n",
       "5           auto          auto               691                      840   \n",
       "...          ...           ...               ...                      ...   \n",
       "1142197     heat          auto               699                      760   \n",
       "1142198     heat          auto               706                      760   \n",
       "1142199     heat          auto               707                      760   \n",
       "1142200     heat          auto               697                      760   \n",
       "1142201     heat          auto               758                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            650            NJ        Middletown   \n",
       "1                            705            NJ        Sayreville   \n",
       "3                            658            NJ      Elmwood Park   \n",
       "4                            677            NJ           Trenton   \n",
       "5                            690            NJ      Mount Laurel   \n",
       "...                          ...           ...               ...   \n",
       "1142197                      700            NJ           Wenonah   \n",
       "1142198                      710            NJ  Evesham Township   \n",
       "1142199                      700            NJ         Glen Rock   \n",
       "1142200                      700            NJ       Cherry Hill   \n",
       "1142201                      760            NJ         Glassboro   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         30              True         False            False   \n",
       "1                         60             False         False            False   \n",
       "3                         90             False         False            False   \n",
       "4                         50             False         False            False   \n",
       "5                         50             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1142197                   70             False         False            False   \n",
       "1142198                    0             False         False             True   \n",
       "1142199                   80             False         False            False   \n",
       "1142200                   50             False         False            False   \n",
       "1142201                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1142197                       Gas  \n",
       "1142198                  Electric  \n",
       "1142199                       Gas  \n",
       "1142200                       Gas  \n",
       "1142201                       Gas  \n",
       "\n",
       "[1066998 rows x 14 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2019.drop(feb_2019[feb_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2019[\"Year\"] = \"2019\"\n",
    "feb_2019[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2019 = feb_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2019_ave = feb_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2019_ave.to_csv(\"data/day/NJ/feb/feb_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2020 = pd.read_csv(\"../data_large/NJ-day/2020-feb-day-NJ.csv\")\n",
    "\n",
    "# feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fb8242e4344a8eb957aae8e71943bbc5641b5c87</td>\n",
       "      <td>2020-02-22 17:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>775</td>\n",
       "      <td>695</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Sicklerville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0735fd45c22def7a595e22134db926c8bc02d5c9</td>\n",
       "      <td>2020-02-20 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>615</td>\n",
       "      <td>620</td>\n",
       "      <td>620</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Milford</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eea162e43c8dd1f3950760333c8b73c940980e97</td>\n",
       "      <td>2020-02-04 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>696</td>\n",
       "      <td>698</td>\n",
       "      <td>698</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West New York</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>268b60939773a9951f49a06e762db62a62422056</td>\n",
       "      <td>2020-02-02 15:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>729</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Collegeville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4358ca81e74c20f993ce976c013caaf6ce2c2151</td>\n",
       "      <td>2020-02-27 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>754</td>\n",
       "      <td>830</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Green Brook</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644439</th>\n",
       "      <td>10b7ae2ac1367cf6022e3e98d43614ce1803b161</td>\n",
       "      <td>2020-02-05 14:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Point Pleasant Boro</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644440</th>\n",
       "      <td>0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea</td>\n",
       "      <td>2020-02-23 19:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>784</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Glassboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644442</th>\n",
       "      <td>aa657e198b0561d5944ff2b8e6c31efdca34b2f1</td>\n",
       "      <td>2020-02-12 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>654</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644443</th>\n",
       "      <td>87a2141d6eaebde08c70b77d8286208d2a4c1ca3</td>\n",
       "      <td>2020-02-10 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>700</td>\n",
       "      <td>760</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Old Bridge</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1644444</th>\n",
       "      <td>71643378b8cea7d3990a8b7d8acb1df217f12aaf</td>\n",
       "      <td>2020-02-11 11:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Plainfield</td>\n",
       "      <td>40</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1555200 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        fb8242e4344a8eb957aae8e71943bbc5641b5c87  2020-02-22 17:20:00 UTC   \n",
       "1        0735fd45c22def7a595e22134db926c8bc02d5c9  2020-02-20 19:10:00 UTC   \n",
       "2        eea162e43c8dd1f3950760333c8b73c940980e97  2020-02-04 17:50:00 UTC   \n",
       "3        268b60939773a9951f49a06e762db62a62422056  2020-02-02 15:05:00 UTC   \n",
       "4        4358ca81e74c20f993ce976c013caaf6ce2c2151  2020-02-27 17:40:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1644439  10b7ae2ac1367cf6022e3e98d43614ce1803b161  2020-02-05 14:00:00 UTC   \n",
       "1644440  0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea  2020-02-23 19:10:00 UTC   \n",
       "1644442  aa657e198b0561d5944ff2b8e6c31efdca34b2f1  2020-02-12 12:35:00 UTC   \n",
       "1644443  87a2141d6eaebde08c70b77d8286208d2a4c1ca3  2020-02-10 16:10:00 UTC   \n",
       "1644444  71643378b8cea7d3990a8b7d8acb1df217f12aaf  2020-02-11 11:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               726                      775   \n",
       "1           heat          auto               615                      620   \n",
       "2           heat          auto               696                      698   \n",
       "3           heat          hold               729                      715   \n",
       "4           auto          hold               754                      830   \n",
       "...          ...           ...               ...                      ...   \n",
       "1644439     heat          auto               675                      760   \n",
       "1644440     heat          hold               784                      760   \n",
       "1644442     auto          auto               654                      760   \n",
       "1644443     heat          auto               700                      760   \n",
       "1644444     auto          hold               706                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            695            NJ         Sicklerville   \n",
       "1                            620            NJ         West Milford   \n",
       "2                            698            NJ        West New York   \n",
       "3                            715            NJ         Collegeville   \n",
       "4                            760            NJ          Green Brook   \n",
       "...                          ...           ...                  ...   \n",
       "1644439                      660            NJ  Point Pleasant Boro   \n",
       "1644440                      760            NJ            Glassboro   \n",
       "1644442                      660            NJ  Egg Harbor Township   \n",
       "1644443                      680            NJ           Old Bridge   \n",
       "1644444                      710            NJ           Plainfield   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         10             False         False            False   \n",
       "1                         45             False         False            False   \n",
       "2                          0              True         False             True   \n",
       "3                          0             False         False            False   \n",
       "4                         30             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1644439                    9             False         False            False   \n",
       "1644440                   20             False         False            False   \n",
       "1644442                   30             False         False            False   \n",
       "1644443                   30             False         False            False   \n",
       "1644444                   40              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                        Electric  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1644439                       Gas  \n",
       "1644440                       Gas  \n",
       "1644442                       Gas  \n",
       "1644443                       Gas  \n",
       "1644444                       Gas  \n",
       "\n",
       "[1555200 rows x 14 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2020.drop(feb_2020[feb_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2020[\"Year\"] = \"2020\"\n",
    "feb_2020[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2020 = feb_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2020_ave = feb_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2020_ave.to_csv(\"data/day/NJ/feb/feb_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 February Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "feb_2021 = pd.read_csv(\"../data_large/NJ-day/2021-feb-day-NJ.csv\")\n",
    "\n",
    "# feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a7f535bb801820946a99ec2c3dac4202029a1ec3</td>\n",
       "      <td>2021-02-27 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>711</td>\n",
       "      <td>711</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>066419ee7f1366fd00952dc760e12b20ee033f68</td>\n",
       "      <td>2021-02-15 12:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>688</td>\n",
       "      <td>688</td>\n",
       "      <td>688</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Readington</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bba5d3aa4933a44e06c7a2b30ff876c6c25e3ccc</td>\n",
       "      <td>2021-02-13 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>738</td>\n",
       "      <td>812</td>\n",
       "      <td>762</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Colts Neck</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1260825fc41783d4513ad7b7e812f9d9d236e82f</td>\n",
       "      <td>2021-02-11 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>713</td>\n",
       "      <td>718</td>\n",
       "      <td>718</td>\n",
       "      <td>NJ</td>\n",
       "      <td>New Providence</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a1348dbfed8425aee385aca041820e0deadadb82</td>\n",
       "      <td>2021-02-21 18:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>670</td>\n",
       "      <td>644</td>\n",
       "      <td>644</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947804</th>\n",
       "      <td>70818e164c5c9b317e1781cf4f976f246c923c5c</td>\n",
       "      <td>2021-02-03 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Atlantic City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947805</th>\n",
       "      <td>82e64c33a0e15163337ea079b31c43d313ce3f90</td>\n",
       "      <td>2021-02-16 16:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Manahawkin</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947806</th>\n",
       "      <td>82e64c33a0e15163337ea079b31c43d313ce3f90</td>\n",
       "      <td>2021-02-28 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>708</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Manahawkin</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947807</th>\n",
       "      <td>41efecf9ac21e53d41762e2209a96cb5fa960706</td>\n",
       "      <td>2021-02-11 19:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>740</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Florham Park</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>947808</th>\n",
       "      <td>a00ded844b2004b4ba8c9c359a3a179e6e7930ce</td>\n",
       "      <td>2021-02-02 19:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>River Edge</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>876294 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       a7f535bb801820946a99ec2c3dac4202029a1ec3  2021-02-27 18:40:00 UTC   \n",
       "1       066419ee7f1366fd00952dc760e12b20ee033f68  2021-02-15 12:55:00 UTC   \n",
       "2       bba5d3aa4933a44e06c7a2b30ff876c6c25e3ccc  2021-02-13 16:15:00 UTC   \n",
       "3       1260825fc41783d4513ad7b7e812f9d9d236e82f  2021-02-11 19:20:00 UTC   \n",
       "4       a1348dbfed8425aee385aca041820e0deadadb82  2021-02-21 18:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "947804  70818e164c5c9b317e1781cf4f976f246c923c5c  2021-02-03 14:20:00 UTC   \n",
       "947805  82e64c33a0e15163337ea079b31c43d313ce3f90  2021-02-16 16:15:00 UTC   \n",
       "947806  82e64c33a0e15163337ea079b31c43d313ce3f90  2021-02-28 13:55:00 UTC   \n",
       "947807  41efecf9ac21e53d41762e2209a96cb5fa960706  2021-02-11 19:00:00 UTC   \n",
       "947808  a00ded844b2004b4ba8c9c359a3a179e6e7930ce  2021-02-02 19:20:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               710                      711   \n",
       "1          heat          hold               688                      688   \n",
       "2          auto          hold               738                      812   \n",
       "3          heat          hold               713                      718   \n",
       "4          heat          hold               670                      644   \n",
       "...         ...           ...               ...                      ...   \n",
       "947804     heat          hold               747                      760   \n",
       "947805     auto          hold               708                      760   \n",
       "947806     auto          hold               708                      760   \n",
       "947807     heat          hold               740                      760   \n",
       "947808     heat          hold               755                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           711            NJ     Cherry Hill   \n",
       "1                           688            NJ      Readington   \n",
       "2                           762            NJ      Colts Neck   \n",
       "3                           718            NJ  New Providence   \n",
       "4                           644            NJ       Ridgewood   \n",
       "...                         ...           ...             ...   \n",
       "947804                      760            NJ   Atlantic City   \n",
       "947805                      710            NJ      Manahawkin   \n",
       "947806                      710            NJ      Manahawkin   \n",
       "947807                      760            NJ    Florham Park   \n",
       "947808                      760            NJ      River Edge   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        50             False         False            False   \n",
       "1                        25             False         False            False   \n",
       "2                        35             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        60             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "947804                   10             False         False             True   \n",
       "947805                    0             False         False            False   \n",
       "947806                    0             False         False            False   \n",
       "947807                    0             False         False            False   \n",
       "947808                   20             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "947804                  Electric  \n",
       "947805                       Gas  \n",
       "947806                       Gas  \n",
       "947807                       Gas  \n",
       "947808                       Gas  \n",
       "\n",
       "[876294 rows x 14 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "feb_2021.drop(feb_2021[feb_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "feb_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "feb_2021[\"Year\"] = \"2021\"\n",
    "feb_2021[\"Month\"] = \"feb\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "feb_2021 = feb_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "feb_2021_ave = feb_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# feb_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "feb_2021_ave.to_csv(\"data/day/NJ/feb/feb_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/feb/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>665.956284</td>\n",
       "      <td>651.311475</td>\n",
       "      <td>641.759563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>706.992278</td>\n",
       "      <td>698.891892</td>\n",
       "      <td>698.084942</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00cfe51c25b825270bc4509dfadca8d4f50a4e99</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Medford</td>\n",
       "      <td>703.496815</td>\n",
       "      <td>706.242038</td>\n",
       "      <td>706.242038</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00cfe51c25b825270bc4509dfadca8d4f50a4e99</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medford</td>\n",
       "      <td>713.112847</td>\n",
       "      <td>715.000000</td>\n",
       "      <td>715.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>01344d14a623a52997206e073b17517a488ddb6c</td>\n",
       "      <td>feb</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Piscataway</td>\n",
       "      <td>747.782609</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1171</th>\n",
       "      <td>fe765ce8408d615dc0bce6dbbc286288933313fd</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Glen Rock</td>\n",
       "      <td>705.411765</td>\n",
       "      <td>707.882353</td>\n",
       "      <td>706.117647</td>\n",
       "      <td>60.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1172</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>696.583001</td>\n",
       "      <td>702.657371</td>\n",
       "      <td>700.134130</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1173</th>\n",
       "      <td>ffacf5dda3b0257b3891ad670f2d1f42187e2277</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>668.722222</td>\n",
       "      <td>664.666667</td>\n",
       "      <td>658.444444</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1174</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>716.250000</td>\n",
       "      <td>721.875000</td>\n",
       "      <td>719.708333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1175</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>feb</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>702.500000</td>\n",
       "      <td>734.000000</td>\n",
       "      <td>703.545455</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7756 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   feb  2017     heat   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   feb  2017     heat   \n",
       "2     00cfe51c25b825270bc4509dfadca8d4f50a4e99   feb  2017     heat   \n",
       "3     00cfe51c25b825270bc4509dfadca8d4f50a4e99   feb  2017     heat   \n",
       "4     01344d14a623a52997206e073b17517a488ddb6c   feb  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1171  fe765ce8408d615dc0bce6dbbc286288933313fd   feb  2021     heat   \n",
       "1172  ff635b7556b71ecf409661242fa988bf1361c850   feb  2021     heat   \n",
       "1173  ffacf5dda3b0257b3891ad670f2d1f42187e2277   feb  2021     heat   \n",
       "1174  ffb88d949cb4d05658d8445b5cc64eb853aea426   feb  2021     heat   \n",
       "1175  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   feb  2021     auto   \n",
       "\n",
       "     CalendarEvent        City  Temperature_ctrl_ave  \\\n",
       "0             auto   Millstone            665.956284   \n",
       "1             hold   Millstone            706.992278   \n",
       "2             auto     Medford            703.496815   \n",
       "3             hold     Medford            713.112847   \n",
       "4             auto  Piscataway            747.782609   \n",
       "...            ...         ...                   ...   \n",
       "1171          hold   Glen Rock            705.411765   \n",
       "1172          hold      newark            696.583001   \n",
       "1173          hold   Oak Ridge            668.722222   \n",
       "1174          hold  Livingston            716.250000   \n",
       "1175          hold     Maywood            702.500000   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      651.311475                   641.759563   \n",
       "1                      698.891892                   698.084942   \n",
       "2                      706.242038                   706.242038   \n",
       "3                      715.000000                   715.000000   \n",
       "4                      750.000000                   680.000000   \n",
       "...                           ...                          ...   \n",
       "1171                   707.882353                   706.117647   \n",
       "1172                   702.657371                   700.134130   \n",
       "1173                   664.666667                   658.444444   \n",
       "1174                   721.875000                   719.708333   \n",
       "1175                   734.000000                   703.545455   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     0.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1171                 60.0             False         False            False  \n",
       "1172                120.0             False         False            False  \n",
       "1173                 99.0              True         False            False  \n",
       "1174                  0.0             False         False            False  \n",
       "1175                 90.0             False         False            False  \n",
       "\n",
       "[7756 rows x 13 columns]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_feb = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/feb/\" + file)\n",
    "    NJ_feb = pd.concat([NJ_feb, df])\n",
    "    \n",
    "NJ_feb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_feb.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_feb.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## March"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## April"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## May"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## June"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2017 = pd.read_csv(\"../data_large/NJ-day/2017-jun-day-NJ.csv\")\n",
    "\n",
    "# jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6f05ed40009c4d2311ccd097b6b6b8563a7de7c6</td>\n",
       "      <td>2017-06-16 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Edgewater</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6c09865f6673a1f668d36eb5bfb2137545ef792f</td>\n",
       "      <td>2017-06-19 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>763</td>\n",
       "      <td>763</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mays Landing</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ebf5db3a589e4589388886191ae1283d764af30d</td>\n",
       "      <td>2017-06-24 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>723</td>\n",
       "      <td>720</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Harrington Park</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ebf5db3a589e4589388886191ae1283d764af30d</td>\n",
       "      <td>2017-06-28 16:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>740</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Harrington Park</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7318e078f87e016d9ae71ff9ab408e4166d29457</td>\n",
       "      <td>2017-06-07 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>708</td>\n",
       "      <td>820</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hillside</td>\n",
       "      <td>46</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523517</th>\n",
       "      <td>c0b793813c1d559990ce01ec9fb2c598e16f367d</td>\n",
       "      <td>2017-06-14 13:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>787</td>\n",
       "      <td>790</td>\n",
       "      <td>720</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523518</th>\n",
       "      <td>feada91765d02a17397b41f77689a4129b6b57d1</td>\n",
       "      <td>2017-06-30 17:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>753</td>\n",
       "      <td>750</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>19</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523519</th>\n",
       "      <td>2aa6414da36a2525d14a9558312566818161044a</td>\n",
       "      <td>2017-06-09 19:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>758</td>\n",
       "      <td>790</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523520</th>\n",
       "      <td>2aa6414da36a2525d14a9558312566818161044a</td>\n",
       "      <td>2017-06-18 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>770</td>\n",
       "      <td>780</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523521</th>\n",
       "      <td>2aa6414da36a2525d14a9558312566818161044a</td>\n",
       "      <td>2017-06-13 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>750</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>508387 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       6f05ed40009c4d2311ccd097b6b6b8563a7de7c6  2017-06-16 12:05:00 UTC   \n",
       "1       6c09865f6673a1f668d36eb5bfb2137545ef792f  2017-06-19 14:30:00 UTC   \n",
       "2       ebf5db3a589e4589388886191ae1283d764af30d  2017-06-24 18:05:00 UTC   \n",
       "3       ebf5db3a589e4589388886191ae1283d764af30d  2017-06-28 16:15:00 UTC   \n",
       "4       7318e078f87e016d9ae71ff9ab408e4166d29457  2017-06-07 17:05:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "523517  c0b793813c1d559990ce01ec9fb2c598e16f367d  2017-06-14 13:55:00 UTC   \n",
       "523518  feada91765d02a17397b41f77689a4129b6b57d1  2017-06-30 17:05:00 UTC   \n",
       "523519  2aa6414da36a2525d14a9558312566818161044a  2017-06-09 19:30:00 UTC   \n",
       "523520  2aa6414da36a2525d14a9558312566818161044a  2017-06-18 14:30:00 UTC   \n",
       "523521  2aa6414da36a2525d14a9558312566818161044a  2017-06-13 17:45:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               742                      743   \n",
       "1          cool          hold               764                      763   \n",
       "2          cool          auto               723                      720   \n",
       "3          cool          auto               734                      740   \n",
       "4          cool          auto               708                      820   \n",
       "...         ...           ...               ...                      ...   \n",
       "523517     cool          auto               787                      790   \n",
       "523518     cool          auto               753                      750   \n",
       "523519     cool          auto               758                      790   \n",
       "523520     cool          auto               764                      770   \n",
       "523521     cool          auto               757                      750   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                           743            NJ            Edgewater   \n",
       "1                           763            NJ         Mays Landing   \n",
       "2                           690            NJ      Harrington Park   \n",
       "3                           710            NJ      Harrington Park   \n",
       "4                           640            NJ             Hillside   \n",
       "...                         ...           ...                  ...   \n",
       "523517                      720            NJ  Egg Harbor Township   \n",
       "523518                      700            NJ  Egg Harbor Township   \n",
       "523519                      670            NJ  Egg Harbor Township   \n",
       "523520                      780            NJ  Egg Harbor Township   \n",
       "523521                      670            NJ  Egg Harbor Township   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        20             False         False             True   \n",
       "1                        30             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                         0             False         False            False   \n",
       "4                        46             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "523517                   57             False         False            False   \n",
       "523518                   19             False         False             True   \n",
       "523519                   50             False         False            False   \n",
       "523520                   50             False         False            False   \n",
       "523521                   50             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                       Electric  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "523517                       Gas  \n",
       "523518                  Electric  \n",
       "523519                       Gas  \n",
       "523520                       Gas  \n",
       "523521                       Gas  \n",
       "\n",
       "[508387 rows x 14 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2017.drop(jun_2017[jun_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2017[\"Year\"] = \"2017\"\n",
    "jun_2017[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2017 = jun_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2017_ave = jun_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2017_ave.to_csv(\"data/day/NJ/jun/jun_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2018 = pd.read_csv(\"../data_large/NJ-day/2018-jun-day-NJ.csv\")\n",
    "\n",
    "# jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c978fda9a1bb73a1a2376a3c821aea5fa80bfa59</td>\n",
       "      <td>2018-06-27 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>707</td>\n",
       "      <td>707</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e624929e3542550e6ec536a903858a5a06ce470c</td>\n",
       "      <td>2018-06-11 10:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>682</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Chatham</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c350527357f7149fcd95f6fa7706010bd6275a53</td>\n",
       "      <td>2018-06-13 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lyndhurst</td>\n",
       "      <td>5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>b6465a892ec628096be7b4a7122b2da0e631cbea</td>\n",
       "      <td>2018-06-13 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>752</td>\n",
       "      <td>784</td>\n",
       "      <td>784</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Orange</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>f358bffffd7738fa2f72b8b4e5a116dd28558946</td>\n",
       "      <td>2018-06-04 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>712</td>\n",
       "      <td>682</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jersey City</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159640</th>\n",
       "      <td>7f4e81372e5d5d60cec5fd02a0474985135dc8e2</td>\n",
       "      <td>2018-06-26 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Salem</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159641</th>\n",
       "      <td>d91e5e7494c24a0ee29e528da6fafbd1d5e099be</td>\n",
       "      <td>2018-06-17 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hoboken</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159642</th>\n",
       "      <td>0735fd45c22def7a595e22134db926c8bc02d5c9</td>\n",
       "      <td>2018-06-26 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Milford</td>\n",
       "      <td>45</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159643</th>\n",
       "      <td>393c8adc3ca3535636e9220b92eade38bffa7853</td>\n",
       "      <td>2018-06-06 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Brunswick</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1159644</th>\n",
       "      <td>ec05bbd1155f3446f85e864ebafefbf45254f75e</td>\n",
       "      <td>2018-06-24 17:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>747</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mullica Hill</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1112108 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        c978fda9a1bb73a1a2376a3c821aea5fa80bfa59  2018-06-27 12:50:00 UTC   \n",
       "2        e624929e3542550e6ec536a903858a5a06ce470c  2018-06-11 10:45:00 UTC   \n",
       "3        c350527357f7149fcd95f6fa7706010bd6275a53  2018-06-13 11:00:00 UTC   \n",
       "4        b6465a892ec628096be7b4a7122b2da0e631cbea  2018-06-13 15:40:00 UTC   \n",
       "5        f358bffffd7738fa2f72b8b4e5a116dd28558946  2018-06-04 19:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1159640  7f4e81372e5d5d60cec5fd02a0474985135dc8e2  2018-06-26 17:20:00 UTC   \n",
       "1159641  d91e5e7494c24a0ee29e528da6fafbd1d5e099be  2018-06-17 17:45:00 UTC   \n",
       "1159642  0735fd45c22def7a595e22134db926c8bc02d5c9  2018-06-26 11:35:00 UTC   \n",
       "1159643  393c8adc3ca3535636e9220b92eade38bffa7853  2018-06-06 19:15:00 UTC   \n",
       "1159644  ec05bbd1155f3446f85e864ebafefbf45254f75e  2018-06-24 17:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           cool          hold               712                      707   \n",
       "2           auto          auto               682                      705   \n",
       "3           cool          hold               728                      726   \n",
       "4           cool          hold               752                      784   \n",
       "5           cool          auto               716                      712   \n",
       "...          ...           ...               ...                      ...   \n",
       "1159640     cool          hold               760                      760   \n",
       "1159641     cool          hold               756                      760   \n",
       "1159642     cool          hold               710                      760   \n",
       "1159643     cool          auto               750                      760   \n",
       "1159644     cool          hold               747                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "1                            707            NJ        Westfield   \n",
       "2                            655            NJ          Chatham   \n",
       "3                            726            NJ        Lyndhurst   \n",
       "4                            784            NJ      West Orange   \n",
       "5                            682            NJ      Jersey City   \n",
       "...                          ...           ...              ...   \n",
       "1159640                      760            NJ            Salem   \n",
       "1159641                      760            NJ          Hoboken   \n",
       "1159642                      760            NJ     West Milford   \n",
       "1159643                      760            NJ  North Brunswick   \n",
       "1159644                      760            NJ     Mullica Hill   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                          5             False         False            False   \n",
       "2                         70             False         False            False   \n",
       "3                          5              True         False             True   \n",
       "4                         25             False         False            False   \n",
       "5                         10             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1159640                   80             False         False            False   \n",
       "1159641                    0             False         False            False   \n",
       "1159642                   45             False         False            False   \n",
       "1159643                   15             False         False            False   \n",
       "1159644                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1159640                       Gas  \n",
       "1159641                       Gas  \n",
       "1159642                       Gas  \n",
       "1159643                       Gas  \n",
       "1159644                       Gas  \n",
       "\n",
       "[1112108 rows x 14 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2018.drop(jun_2018[jun_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2018[\"Year\"] = \"2018\"\n",
    "jun_2018[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2018 = jun_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2018_ave = jun_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2018_ave.to_csv(\"data/day/NJ/jun/jun_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2019 = pd.read_csv(\"../data_large/NJ-day/2019-jun-day-NJ.csv\")\n",
    "\n",
    "# jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bddd5cea65be1b55e157533cbc4f13c8f506154b</td>\n",
       "      <td>2019-06-01 14:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>724</td>\n",
       "      <td>680</td>\n",
       "      <td>657</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mount Laurel</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2abf248171a5a0f1023ce7910e4e63cb6cd47c30</td>\n",
       "      <td>2019-06-26 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>816</td>\n",
       "      <td>705</td>\n",
       "      <td>655</td>\n",
       "      <td>NJ</td>\n",
       "      <td>N Wildwood</td>\n",
       "      <td>15</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9cd0112e4c82cc0c3690ea5b0c60e1c3e838b024</td>\n",
       "      <td>2019-06-01 14:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Cape May</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>faf82d5780c46cd24e15d66fba2106fb21d2c115</td>\n",
       "      <td>2019-06-06 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>773</td>\n",
       "      <td>770</td>\n",
       "      <td>626</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West New York</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1f98c42a78f42e25e5b0c2dec6b5377e988dcf91</td>\n",
       "      <td>2019-06-11 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>705</td>\n",
       "      <td>726</td>\n",
       "      <td>726</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Holmdel</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1756600</th>\n",
       "      <td>b330f8153b5c97be1d0460742f0cc5ed0711208e</td>\n",
       "      <td>2019-06-16 19:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Vineland</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1756601</th>\n",
       "      <td>d2b4b0edff4d937154c31af3dbc4c54bbccb5179</td>\n",
       "      <td>2019-06-10 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Evesham</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1756602</th>\n",
       "      <td>d02414f35a6a5293acd7e4826936afd4088d7f09</td>\n",
       "      <td>2019-06-23 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hoboken</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1756603</th>\n",
       "      <td>d59a49513285b9ecf44b1e3774c0fff492c7c4a0</td>\n",
       "      <td>2019-06-30 15:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>775</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Newton</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1756604</th>\n",
       "      <td>d59a49513285b9ecf44b1e3774c0fff492c7c4a0</td>\n",
       "      <td>2019-06-30 15:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>766</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Newton</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1683834 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        bddd5cea65be1b55e157533cbc4f13c8f506154b  2019-06-01 14:20:00 UTC   \n",
       "1        2abf248171a5a0f1023ce7910e4e63cb6cd47c30  2019-06-26 12:10:00 UTC   \n",
       "2        9cd0112e4c82cc0c3690ea5b0c60e1c3e838b024  2019-06-01 14:25:00 UTC   \n",
       "3        faf82d5780c46cd24e15d66fba2106fb21d2c115  2019-06-06 15:40:00 UTC   \n",
       "5        1f98c42a78f42e25e5b0c2dec6b5377e988dcf91  2019-06-11 14:15:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1756600  b330f8153b5c97be1d0460742f0cc5ed0711208e  2019-06-16 19:35:00 UTC   \n",
       "1756601  d2b4b0edff4d937154c31af3dbc4c54bbccb5179  2019-06-10 12:55:00 UTC   \n",
       "1756602  d02414f35a6a5293acd7e4826936afd4088d7f09  2019-06-23 18:55:00 UTC   \n",
       "1756603  d59a49513285b9ecf44b1e3774c0fff492c7c4a0  2019-06-30 15:05:00 UTC   \n",
       "1756604  d59a49513285b9ecf44b1e3774c0fff492c7c4a0  2019-06-30 15:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               724                      680   \n",
       "1           cool          auto               816                      705   \n",
       "2           cool          hold               706                      705   \n",
       "3           auto          hold               773                      770   \n",
       "5           cool          hold               705                      726   \n",
       "...          ...           ...               ...                      ...   \n",
       "1756600     cool          hold               766                      760   \n",
       "1756601     cool          hold               720                      760   \n",
       "1756602     cool          hold               760                      760   \n",
       "1756603     cool          hold               775                      760   \n",
       "1756604     cool          hold               766                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            657            NJ    Mount Laurel   \n",
       "1                            655            NJ      N Wildwood   \n",
       "2                            705            NJ  North Cape May   \n",
       "3                            626            NJ   West New York   \n",
       "5                            726            NJ         Holmdel   \n",
       "...                          ...           ...             ...   \n",
       "1756600                      760            NJ        Vineland   \n",
       "1756601                      760            NJ         Evesham   \n",
       "1756602                      760            NJ         Hoboken   \n",
       "1756603                      760            NJ          Newton   \n",
       "1756604                      760            NJ          Newton   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "1                         15              True         False            False   \n",
       "2                         27             False         False            False   \n",
       "3                          0              True         False             True   \n",
       "5                         50             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1756600                   19              True         False            False   \n",
       "1756601                    9              True         False            False   \n",
       "1756602                   10             False         False            False   \n",
       "1756603                   40             False         False            False   \n",
       "1756604                   40             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1756600                       Gas  \n",
       "1756601                       Gas  \n",
       "1756602                       Gas  \n",
       "1756603                       Gas  \n",
       "1756604                       Gas  \n",
       "\n",
       "[1683834 rows x 14 columns]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2019.drop(jun_2019[jun_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2019[\"Year\"] = \"2019\"\n",
    "jun_2019[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2019 = jun_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2019_ave = jun_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2019_ave.to_csv(\"data/day/NJ/jun/jun_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2020 = pd.read_csv(\"../data_large/NJ-day/2020-jun-day-NJ.csv\")\n",
    "\n",
    "# jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2f7fd3de7c0ea58e68e58efae2493e3ab1b65e58</td>\n",
       "      <td>2020-06-09 13:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>709</td>\n",
       "      <td>670</td>\n",
       "      <td>677</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mt. Laurel</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e1be0f8a7ad278c797f4715a492a1872cc015636</td>\n",
       "      <td>2020-06-17 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>680</td>\n",
       "      <td>677</td>\n",
       "      <td>677</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Runnemede</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3584d50be0c0ac7fa814da32446fca6c212a6aba</td>\n",
       "      <td>2020-06-09 13:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montclair</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>55fe049d27d78ce50052a06c5a2ec4313e00ff61</td>\n",
       "      <td>2020-06-16 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>737</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a584a8df86b73288dc9e904ca3d9491836f82abc</td>\n",
       "      <td>2020-06-26 12:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>761</td>\n",
       "      <td>617</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Union</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1698140</th>\n",
       "      <td>f0943a6be8a30d9ae76dbeb5574b339273be31b1</td>\n",
       "      <td>2020-06-10 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cranford</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1698141</th>\n",
       "      <td>84d188e0b9f8fe4036682ae0d5c2b152124b2274</td>\n",
       "      <td>2020-06-28 15:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ventnor City</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1698142</th>\n",
       "      <td>57f42c5925331c03754ca37b6cb8071fd9295e07</td>\n",
       "      <td>2020-06-14 17:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Freehold</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1698143</th>\n",
       "      <td>287f02374ce923dfd3bb89e40c480c600a9520e7</td>\n",
       "      <td>2020-06-19 15:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Franklin Park</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1698144</th>\n",
       "      <td>4c1603cd9c6aaf262685ed50aedaaf208f761afd</td>\n",
       "      <td>2020-06-26 17:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1650114 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        2f7fd3de7c0ea58e68e58efae2493e3ab1b65e58  2020-06-09 13:45:00 UTC   \n",
       "1        e1be0f8a7ad278c797f4715a492a1872cc015636  2020-06-17 19:55:00 UTC   \n",
       "2        3584d50be0c0ac7fa814da32446fca6c212a6aba  2020-06-09 13:55:00 UTC   \n",
       "3        55fe049d27d78ce50052a06c5a2ec4313e00ff61  2020-06-16 11:00:00 UTC   \n",
       "4        a584a8df86b73288dc9e904ca3d9491836f82abc  2020-06-26 12:35:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1698140  f0943a6be8a30d9ae76dbeb5574b339273be31b1  2020-06-10 19:00:00 UTC   \n",
       "1698141  84d188e0b9f8fe4036682ae0d5c2b152124b2274  2020-06-28 15:45:00 UTC   \n",
       "1698142  57f42c5925331c03754ca37b6cb8071fd9295e07  2020-06-14 17:20:00 UTC   \n",
       "1698143  287f02374ce923dfd3bb89e40c480c600a9520e7  2020-06-19 15:20:00 UTC   \n",
       "1698144  4c1603cd9c6aaf262685ed50aedaaf208f761afd  2020-06-26 17:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               709                      670   \n",
       "1           cool          hold               680                      677   \n",
       "2           auto          hold               681                      695   \n",
       "3           cool          hold               737                      752   \n",
       "4           auto          hold               764                      761   \n",
       "...          ...           ...               ...                      ...   \n",
       "1698140     cool          auto               767                      760   \n",
       "1698141     cool          hold               759                      760   \n",
       "1698142     cool          auto               734                      760   \n",
       "1698143     cool          auto               764                      760   \n",
       "1698144     cool          auto               756                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState           City  \\\n",
       "0                            677            NJ     Mt. Laurel   \n",
       "1                            677            NJ      Runnemede   \n",
       "2                            645            NJ      Montclair   \n",
       "3                            752            NJ      Westfield   \n",
       "4                            617            NJ          Union   \n",
       "...                          ...           ...            ...   \n",
       "1698140                      760            NJ       Cranford   \n",
       "1698141                      760            NJ   Ventnor City   \n",
       "1698142                      760            NJ       Freehold   \n",
       "1698143                      760            NJ  Franklin Park   \n",
       "1698144                      760            NJ      Millstone   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "1                         80             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          5             False         False            False   \n",
       "4                          5             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1698140                   99              True         False            False   \n",
       "1698141                    9             False         False            False   \n",
       "1698142                   10             False         False            False   \n",
       "1698143                  118              True         False            False   \n",
       "1698144                   20             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1698140                       Gas  \n",
       "1698141                       Gas  \n",
       "1698142                       Gas  \n",
       "1698143                       Gas  \n",
       "1698144                       Gas  \n",
       "\n",
       "[1650114 rows x 14 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2020.drop(jun_2020[jun_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2020[\"Year\"] = \"2020\"\n",
    "jun_2020[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2020 = jun_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2020_ave = jun_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2020_ave.to_csv(\"data/day/NJ/jun/jun_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 June Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jun_2021 = pd.read_csv(\"../data_large/NJ-day/2021-jun-day-NJ.csv\")\n",
    "\n",
    "# jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>91ccec7bc792576ac0589fd5fd9261601671668f</td>\n",
       "      <td>2021-06-08 18:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>810</td>\n",
       "      <td>809</td>\n",
       "      <td>809</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Leonia</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>06a03eaa78e22728cc2eb752b910457ed5b108ce</td>\n",
       "      <td>2021-06-08 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>694</td>\n",
       "      <td>694</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Stewartsville</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>89053abc9571ee3f6824c4a9dee660a640b8d5c3</td>\n",
       "      <td>2021-06-06 12:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>727</td>\n",
       "      <td>687</td>\n",
       "      <td>687</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Long Branch</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>887d3c611f098f2a969ddac52e32d42d4feea641</td>\n",
       "      <td>2021-06-03 17:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>695</td>\n",
       "      <td>692</td>\n",
       "      <td>632</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Moorestown</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>bba5d3aa4933a44e06c7a2b30ff876c6c25e3ccc</td>\n",
       "      <td>2021-06-30 19:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>774</td>\n",
       "      <td>712</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Colts Neck</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995956</th>\n",
       "      <td>166b48b5154d91187b73316eaa6e59154c1854c2</td>\n",
       "      <td>2021-06-07 10:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995957</th>\n",
       "      <td>3c13c0b947f008e20c7fe2bc7370f16a9130e1a0</td>\n",
       "      <td>2021-06-27 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ventnor City</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995958</th>\n",
       "      <td>ba0791c82ba05dce044df8c5afb240523811e248</td>\n",
       "      <td>2021-06-24 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>751</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>East Brunswick</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995959</th>\n",
       "      <td>99108fa4c6bead747a7cb820badbc28eab58b7d3</td>\n",
       "      <td>2021-06-11 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>743</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Voorhees</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995960</th>\n",
       "      <td>8b8f1e563136f9119a1ea5b10378f8d7ebc8af18</td>\n",
       "      <td>2021-06-22 14:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>961035 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       91ccec7bc792576ac0589fd5fd9261601671668f  2021-06-08 18:30:00 UTC   \n",
       "2       06a03eaa78e22728cc2eb752b910457ed5b108ce  2021-06-08 14:40:00 UTC   \n",
       "4       89053abc9571ee3f6824c4a9dee660a640b8d5c3  2021-06-06 12:40:00 UTC   \n",
       "6       887d3c611f098f2a969ddac52e32d42d4feea641  2021-06-03 17:50:00 UTC   \n",
       "7       bba5d3aa4933a44e06c7a2b30ff876c6c25e3ccc  2021-06-30 19:25:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "995956  166b48b5154d91187b73316eaa6e59154c1854c2  2021-06-07 10:30:00 UTC   \n",
       "995957  3c13c0b947f008e20c7fe2bc7370f16a9130e1a0  2021-06-27 18:20:00 UTC   \n",
       "995958  ba0791c82ba05dce044df8c5afb240523811e248  2021-06-24 16:45:00 UTC   \n",
       "995959  99108fa4c6bead747a7cb820badbc28eab58b7d3  2021-06-11 12:05:00 UTC   \n",
       "995960  8b8f1e563136f9119a1ea5b10378f8d7ebc8af18  2021-06-22 14:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               810                      809   \n",
       "2          cool          hold               697                      694   \n",
       "4          cool          hold               727                      687   \n",
       "6          auto          hold               695                      692   \n",
       "7          auto          hold               778                      774   \n",
       "...         ...           ...               ...                      ...   \n",
       "995956     cool          hold               756                      760   \n",
       "995957     cool          hold               762                      760   \n",
       "995958     cool          hold               751                      760   \n",
       "995959     cool          hold               743                      760   \n",
       "995960     cool          hold               750                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                           809            NJ          Leonia   \n",
       "2                           694            NJ   Stewartsville   \n",
       "4                           687            NJ     Long Branch   \n",
       "6                           632            NJ      Moorestown   \n",
       "7                           712            NJ      Colts Neck   \n",
       "...                         ...           ...             ...   \n",
       "995956                      760            NJ       Ridgewood   \n",
       "995957                      760            NJ    Ventnor City   \n",
       "995958                      760            NJ  East Brunswick   \n",
       "995959                      760            NJ        Voorhees   \n",
       "995960                      760            NJ     Cherry Hill   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        90             False         False            False   \n",
       "2                        10             False         False            False   \n",
       "4                         0             False         False            False   \n",
       "6                        16             False         False            False   \n",
       "7                        35             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "995956                    0             False         False            False   \n",
       "995957                    9              True         False            False   \n",
       "995958                    0              True         False            False   \n",
       "995959                   30             False         False            False   \n",
       "995960                    9             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "4                            Gas  \n",
       "6                            Gas  \n",
       "7                            Gas  \n",
       "...                          ...  \n",
       "995956                       Gas  \n",
       "995957                       Gas  \n",
       "995958                       Gas  \n",
       "995959                       Gas  \n",
       "995960                       Gas  \n",
       "\n",
       "[961035 rows x 14 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jun_2021.drop(jun_2021[jun_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jun_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jun_2021[\"Year\"] = \"2021\"\n",
    "jun_2021[\"Month\"] = \"jun\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jun_2021 = jun_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jun_2021_ave = jun_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jun_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jun_2021_ave.to_csv(\"data/day/NJ/jun/jun_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/jun/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>762.734266</td>\n",
       "      <td>824.440559</td>\n",
       "      <td>692.370629</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>766.899225</td>\n",
       "      <td>776.651163</td>\n",
       "      <td>775.478036</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00cfe51c25b825270bc4509dfadca8d4f50a4e99</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Medford</td>\n",
       "      <td>724.961538</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>704.910543</td>\n",
       "      <td>703.948083</td>\n",
       "      <td>653.891374</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>jun</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>700.143667</td>\n",
       "      <td>729.039698</td>\n",
       "      <td>669.370510</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1459</th>\n",
       "      <td>fe66c5051b7e13329c0c52a696a10794fa5efd7d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Swedesboro</td>\n",
       "      <td>730.272727</td>\n",
       "      <td>721.806061</td>\n",
       "      <td>720.733333</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460</th>\n",
       "      <td>feada91765d02a17397b41f77689a4129b6b57d1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>711.272727</td>\n",
       "      <td>685.303030</td>\n",
       "      <td>685.363636</td>\n",
       "      <td>19.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1461</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>751.298401</td>\n",
       "      <td>760.071048</td>\n",
       "      <td>759.863233</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1462</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>774.692308</td>\n",
       "      <td>768.307692</td>\n",
       "      <td>768.307692</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1463</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>716.130869</td>\n",
       "      <td>715.254745</td>\n",
       "      <td>694.575425</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9673 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   jun  2017     cool   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   jun  2017     cool   \n",
       "2     00cfe51c25b825270bc4509dfadca8d4f50a4e99   jun  2017     heat   \n",
       "3     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   jun  2017     auto   \n",
       "4     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   jun  2017     auto   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1459  fe66c5051b7e13329c0c52a696a10794fa5efd7d   jun  2021     cool   \n",
       "1460  feada91765d02a17397b41f77689a4129b6b57d1   jun  2021     cool   \n",
       "1461  ff635b7556b71ecf409661242fa988bf1361c850   jun  2021     cool   \n",
       "1462  ffb88d949cb4d05658d8445b5cc64eb853aea426   jun  2021     cool   \n",
       "1463  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   jun  2021     auto   \n",
       "\n",
       "     CalendarEvent                 City  Temperature_ctrl_ave  \\\n",
       "0             auto            Millstone            762.734266   \n",
       "1             hold            Millstone            766.899225   \n",
       "2             auto              Medford            724.961538   \n",
       "3             auto         Hillsborough            704.910543   \n",
       "4             hold         Hillsborough            700.143667   \n",
       "...            ...                  ...                   ...   \n",
       "1459          hold           Swedesboro            730.272727   \n",
       "1460          hold  Egg Harbor Township            711.272727   \n",
       "1461          hold               newark            751.298401   \n",
       "1462          hold           Livingston            774.692308   \n",
       "1463          hold              Maywood            716.130869   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      824.440559                   692.370629   \n",
       "1                      776.651163                   775.478036   \n",
       "2                      720.000000                   720.000000   \n",
       "3                      703.948083                   653.891374   \n",
       "4                      729.039698                   669.370510   \n",
       "...                           ...                          ...   \n",
       "1459                   721.806061                   720.733333   \n",
       "1460                   685.303030                   685.363636   \n",
       "1461                   760.071048                   759.863233   \n",
       "1462                   768.307692                   768.307692   \n",
       "1463                   715.254745                   694.575425   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     5.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1459                 10.0              True         False            False  \n",
       "1460                 19.0             False         False             True  \n",
       "1461                120.0             False         False            False  \n",
       "1462                  0.0             False         False            False  \n",
       "1463                 90.0             False         False            False  \n",
       "\n",
       "[9673 rows x 13 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_jun = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/jun/\" + file)\n",
    "    NJ_jun = pd.concat([NJ_jun, df])\n",
    "    \n",
    "NJ_jun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_jun.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_jun.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## July"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2017 = pd.read_csv(\"../data_large/NJ-day/2017-jul-day-NJ.csv\")\n",
    "\n",
    "# jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cb7d514907a4874c1a37060fcb43bd961ea7812</td>\n",
       "      <td>2017-07-26 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>789</td>\n",
       "      <td>800</td>\n",
       "      <td>800</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mullica Hill</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c815289897954a071941d2d1a32759f483d695c1</td>\n",
       "      <td>2017-07-02 15:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>740</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Willingboro</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>71e4ab391d61f4e94db93e360d6d075e8cfe0017</td>\n",
       "      <td>2017-07-13 13:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Bergen</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>936ada79ee18c1afe2eb969ead5ee8fca773b08c</td>\n",
       "      <td>2017-07-07 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Secaucus</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>57b6772e7fab5a780378f4fe5f63d718e020ba5e</td>\n",
       "      <td>2017-07-28 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>699</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>River Vale</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680574</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-07-14 17:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>778</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680575</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-07-30 19:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680576</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-07-20 12:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680577</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-07-05 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680578</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-07-14 14:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>769</td>\n",
       "      <td>770</td>\n",
       "      <td>770</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>660391 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       3cb7d514907a4874c1a37060fcb43bd961ea7812  2017-07-26 14:35:00 UTC   \n",
       "1       c815289897954a071941d2d1a32759f483d695c1  2017-07-02 15:30:00 UTC   \n",
       "2       71e4ab391d61f4e94db93e360d6d075e8cfe0017  2017-07-13 13:10:00 UTC   \n",
       "3       936ada79ee18c1afe2eb969ead5ee8fca773b08c  2017-07-07 18:35:00 UTC   \n",
       "4       57b6772e7fab5a780378f4fe5f63d718e020ba5e  2017-07-28 10:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "680574  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-07-14 17:55:00 UTC   \n",
       "680575  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-07-30 19:25:00 UTC   \n",
       "680576  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-07-20 12:55:00 UTC   \n",
       "680577  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-07-05 14:50:00 UTC   \n",
       "680578  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-07-14 14:05:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          auto               789                      800   \n",
       "1          cool          auto               750                      750   \n",
       "2          cool          hold               759                      750   \n",
       "3          auto          auto               761                      760   \n",
       "4          cool          hold               699                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "680574     cool          hold               778                      770   \n",
       "680575     cool          hold               769                      770   \n",
       "680576     cool          hold               759                      760   \n",
       "680577     cool          hold               763                      770   \n",
       "680578     cool          hold               769                      770   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                  City  \\\n",
       "0                           800            NJ          Mullica Hill   \n",
       "1                           740            NJ           Willingboro   \n",
       "2                           750            NJ          North Bergen   \n",
       "3                           700            NJ              Secaucus   \n",
       "4                           700            NJ            River Vale   \n",
       "...                         ...           ...                   ...   \n",
       "680574                      770            NJ  Cape May Court House   \n",
       "680575                      770            NJ  Cape May Court House   \n",
       "680576                      760            NJ  Cape May Court House   \n",
       "680577                      770            NJ  Cape May Court House   \n",
       "680578                      770            NJ  Cape May Court House   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        25              True         False            False   \n",
       "1                        30             False         False            False   \n",
       "2                         0             False         False            False   \n",
       "3                        40             False         False            False   \n",
       "4                         5             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "680574                    0             False         False            False   \n",
       "680575                    0             False         False            False   \n",
       "680576                    0             False         False            False   \n",
       "680577                    0             False         False            False   \n",
       "680578                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "680574                       Gas  \n",
       "680575                       Gas  \n",
       "680576                       Gas  \n",
       "680577                       Gas  \n",
       "680578                       Gas  \n",
       "\n",
       "[660391 rows x 14 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2017.drop(jul_2017[jul_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2017[\"Year\"] = \"2017\"\n",
    "jul_2017[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2017 = jul_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2017_ave = jul_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2017_ave.to_csv(\"data/day/NJ/jul/jul_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2018 = pd.read_csv(\"../data_large/NJ-day/2018-jul-day-NJ.csv\")\n",
    "\n",
    "# jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aed225d4198c685deca8020969a789e1aa6d36cb</td>\n",
       "      <td>2018-07-05 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>684</td>\n",
       "      <td>684</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hawthorne</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5e4f3de0c4aa6aea8e9da8ba07bf8e669a648b30</td>\n",
       "      <td>2018-07-22 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>708</td>\n",
       "      <td>684</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Palisades Park</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e7c2709c4db263deb49322593636f0d83aa7cedf</td>\n",
       "      <td>2018-07-22 16:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>792</td>\n",
       "      <td>650</td>\n",
       "      <td>635</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Bernards</td>\n",
       "      <td>27</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9e3bf300e5ed900af7c553d657a1dc3f2e84b55a</td>\n",
       "      <td>2018-07-30 19:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>790</td>\n",
       "      <td>788</td>\n",
       "      <td>788</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jersey City</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>97718cb4671bb743659fde0b29c83393c402bda4</td>\n",
       "      <td>2018-07-14 19:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>718</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Howell</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423994</th>\n",
       "      <td>dfbe11b04ab2af18f3e3e80e5b0725b80c1f681f</td>\n",
       "      <td>2018-07-22 14:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>713</td>\n",
       "      <td>720</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Paramus</td>\n",
       "      <td>28</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423995</th>\n",
       "      <td>650cebbaad3dff0df534495c9623c480a27c7b72</td>\n",
       "      <td>2018-07-14 12:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>722</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Clifton</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423996</th>\n",
       "      <td>14c2199c5f222f738c33922a0cdcdd6272737834</td>\n",
       "      <td>2018-07-30 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Blackwood</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423997</th>\n",
       "      <td>0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea</td>\n",
       "      <td>2018-07-28 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Glassboro</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1423998</th>\n",
       "      <td>77532ddff2fb4c20da260594c9f6a4f7fe3e2e39</td>\n",
       "      <td>2018-07-23 18:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>754</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Dumont</td>\n",
       "      <td>57</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1375582 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "1        aed225d4198c685deca8020969a789e1aa6d36cb  2018-07-05 15:35:00 UTC   \n",
       "2        5e4f3de0c4aa6aea8e9da8ba07bf8e669a648b30  2018-07-22 17:30:00 UTC   \n",
       "3        e7c2709c4db263deb49322593636f0d83aa7cedf  2018-07-22 16:45:00 UTC   \n",
       "4        9e3bf300e5ed900af7c553d657a1dc3f2e84b55a  2018-07-30 19:45:00 UTC   \n",
       "5        97718cb4671bb743659fde0b29c83393c402bda4  2018-07-14 19:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1423994  dfbe11b04ab2af18f3e3e80e5b0725b80c1f681f  2018-07-22 14:35:00 UTC   \n",
       "1423995  650cebbaad3dff0df534495c9623c480a27c7b72  2018-07-14 12:50:00 UTC   \n",
       "1423996  14c2199c5f222f738c33922a0cdcdd6272737834  2018-07-30 17:40:00 UTC   \n",
       "1423997  0dc05bbafb3d7026d8a8742ebdb0376c0cb723ea  2018-07-28 19:40:00 UTC   \n",
       "1423998  77532ddff2fb4c20da260594c9f6a4f7fe3e2e39  2018-07-23 18:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "1           cool          hold               686                      684   \n",
       "2           cool          hold               701                      708   \n",
       "3           heat          hold               792                      650   \n",
       "4           cool          auto               790                      788   \n",
       "5           cool          hold               718                      715   \n",
       "...          ...           ...               ...                      ...   \n",
       "1423994     cool          auto               713                      720   \n",
       "1423995     cool          hold               722                      760   \n",
       "1423996     cool          hold               763                      760   \n",
       "1423997     cool          hold               761                      760   \n",
       "1423998     cool          auto               754                      750   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "1                            684            NJ       Hawthorne   \n",
       "2                            684            NJ  Palisades Park   \n",
       "3                            635            NJ        Bernards   \n",
       "4                            788            NJ     Jersey City   \n",
       "5                            715            NJ          Howell   \n",
       "...                          ...           ...             ...   \n",
       "1423994                      760            NJ         Paramus   \n",
       "1423995                      760            NJ         Clifton   \n",
       "1423996                      760            NJ       Blackwood   \n",
       "1423997                      760            NJ       Glassboro   \n",
       "1423998                      760            NJ          Dumont   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "1                         60             False         False            False   \n",
       "2                         60              True         False            False   \n",
       "3                         27             False         False            False   \n",
       "4                         60             False         False            False   \n",
       "5                         20             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1423994                   28             False         False            False   \n",
       "1423995                    0             False         False            False   \n",
       "1423996                   40             False         False            False   \n",
       "1423997                   20             False         False            False   \n",
       "1423998                   57             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "5                             Gas  \n",
       "...                           ...  \n",
       "1423994                       Gas  \n",
       "1423995                       Gas  \n",
       "1423996                       Gas  \n",
       "1423997                       Gas  \n",
       "1423998                       Gas  \n",
       "\n",
       "[1375582 rows x 14 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2018.drop(jul_2018[jul_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2018[\"Year\"] = \"2018\"\n",
    "jul_2018[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2018 = jul_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2018_ave = jul_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2018_ave.to_csv(\"data/day/NJ/jul/jul_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2019 = pd.read_csv(\"../data_large/NJ-day/2019-jul-day-NJ.csv\")\n",
    "\n",
    "# jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>da73bce9ec82573f791be6ad0ddaa86f541f903a</td>\n",
       "      <td>2019-07-11 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>669</td>\n",
       "      <td>670</td>\n",
       "      <td>625</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Bernards</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>471571826f5dc2062ce7b1694699a164f65b5428</td>\n",
       "      <td>2019-07-07 17:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>655</td>\n",
       "      <td>642</td>\n",
       "      <td>642</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hawthorne</td>\n",
       "      <td>9</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0d36823a876cdc9cccd7a304aa1b83ba96340d75</td>\n",
       "      <td>2019-07-14 18:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>729</td>\n",
       "      <td>NJ</td>\n",
       "      <td>edison</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e3a5276c1b757eeedbbe0f523c4601b67e7bd710</td>\n",
       "      <td>2019-07-13 13:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>810</td>\n",
       "      <td>749</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Whiting</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d93f6b92b171e1393fe1811bf6125081feab12fe</td>\n",
       "      <td>2019-07-06 17:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>725</td>\n",
       "      <td>615</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Clarksboro</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090512</th>\n",
       "      <td>d99dea0e4e63941ee49154886f66b2b512b3d244</td>\n",
       "      <td>2019-07-27 13:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lyndhurst Township</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090513</th>\n",
       "      <td>43bfcc8dca763e62e17c0649b4fa2302ba51b788</td>\n",
       "      <td>2019-07-29 17:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montclair</td>\n",
       "      <td>85</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090514</th>\n",
       "      <td>acae27e79f228517afb36247f0c33e6ee1554a70</td>\n",
       "      <td>2019-07-09 09:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>739</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jackson Township</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090515</th>\n",
       "      <td>393c8adc3ca3535636e9220b92eade38bffa7853</td>\n",
       "      <td>2019-07-11 11:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Brunswick</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2090516</th>\n",
       "      <td>166b48b5154d91187b73316eaa6e59154c1854c2</td>\n",
       "      <td>2019-07-08 18:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2009661 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        da73bce9ec82573f791be6ad0ddaa86f541f903a  2019-07-11 11:45:00 UTC   \n",
       "1        471571826f5dc2062ce7b1694699a164f65b5428  2019-07-07 17:10:00 UTC   \n",
       "2        0d36823a876cdc9cccd7a304aa1b83ba96340d75  2019-07-14 18:40:00 UTC   \n",
       "3        e3a5276c1b757eeedbbe0f523c4601b67e7bd710  2019-07-13 13:00:00 UTC   \n",
       "4        d93f6b92b171e1393fe1811bf6125081feab12fe  2019-07-06 17:05:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2090512  d99dea0e4e63941ee49154886f66b2b512b3d244  2019-07-27 13:20:00 UTC   \n",
       "2090513  43bfcc8dca763e62e17c0649b4fa2302ba51b788  2019-07-29 17:45:00 UTC   \n",
       "2090514  acae27e79f228517afb36247f0c33e6ee1554a70  2019-07-09 09:35:00 UTC   \n",
       "2090515  393c8adc3ca3535636e9220b92eade38bffa7853  2019-07-11 11:50:00 UTC   \n",
       "2090516  166b48b5154d91187b73316eaa6e59154c1854c2  2019-07-08 18:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               669                      670   \n",
       "1           cool          hold               655                      642   \n",
       "2           cool          hold               759                      760   \n",
       "3           cool          auto               757                      810   \n",
       "4           auto          hold               728                      725   \n",
       "...          ...           ...               ...                      ...   \n",
       "2090512     cool          hold               765                      760   \n",
       "2090513     cool          auto               760                      760   \n",
       "2090514     cool          hold               739                      760   \n",
       "2090515     cool          hold               760                      760   \n",
       "2090516     cool          auto               761                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                City  \\\n",
       "0                            625            NJ            Bernards   \n",
       "1                            642            NJ           Hawthorne   \n",
       "2                            729            NJ              edison   \n",
       "3                            749            NJ             Whiting   \n",
       "4                            615            NJ          Clarksboro   \n",
       "...                          ...           ...                 ...   \n",
       "2090512                      760            NJ  Lyndhurst Township   \n",
       "2090513                      760            NJ           Montclair   \n",
       "2090514                      760            NJ    Jackson Township   \n",
       "2090515                      760            NJ     North Brunswick   \n",
       "2090516                      760            NJ           Ridgewood   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          7             False         False            False   \n",
       "1                          9              True         False            False   \n",
       "2                          5             False         False            False   \n",
       "3                         40             False         False             True   \n",
       "4                         50              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "2090512                   15             False         False            False   \n",
       "2090513                   85              True         False            False   \n",
       "2090514                    0             False         False            False   \n",
       "2090515                   15             False         False            False   \n",
       "2090516                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "2090512                       Gas  \n",
       "2090513                       Gas  \n",
       "2090514                       Gas  \n",
       "2090515                       Gas  \n",
       "2090516                       Gas  \n",
       "\n",
       "[2009661 rows x 14 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2019.drop(jul_2019[jul_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2019[\"Year\"] = \"2019\"\n",
    "jul_2019[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2019 = jul_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2019_ave = jul_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2019_ave.to_csv(\"data/day/NJ/jul/jul_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2020 = pd.read_csv(\"../data_large/NJ-day/2020-jul-day-NJ.csv\")\n",
    "\n",
    "# jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7d806f8c16b04b89c5050be6fe34fe46725284c1</td>\n",
       "      <td>2020-07-16 11:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>795</td>\n",
       "      <td>820</td>\n",
       "      <td>733</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ocean City</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9f886ba821b603f7ee8b7743b1c67e74977f6427</td>\n",
       "      <td>2020-07-05 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>728</td>\n",
       "      <td>728</td>\n",
       "      <td>703</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Bayonne</td>\n",
       "      <td>10</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f4bf9e58556aef964b509d6cb8d16e4044df48ea</td>\n",
       "      <td>2020-07-12 14:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>761</td>\n",
       "      <td>761</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0b68615a3380664a93a5b796eb46132fa5ef699d</td>\n",
       "      <td>2020-07-10 17:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>742</td>\n",
       "      <td>752</td>\n",
       "      <td>752</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3370ca7e2d9ece26c820bf1b819fadb3cd46f1b1</td>\n",
       "      <td>2020-07-08 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>779</td>\n",
       "      <td>780</td>\n",
       "      <td>747</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Chesterfield</td>\n",
       "      <td>120</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937972</th>\n",
       "      <td>745f2411605d497ab6e2fc336f18573206e09db3</td>\n",
       "      <td>2020-07-07 16:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>771</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Brunswick</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937973</th>\n",
       "      <td>f99a051f8bea5fb3bc5db59c6266a699ac680f67</td>\n",
       "      <td>2020-07-24 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jersey City</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937974</th>\n",
       "      <td>9c9287511ca977dfd2ae1b2811e90e9c00c66485</td>\n",
       "      <td>2020-07-14 19:00:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>765</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hazlet</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937975</th>\n",
       "      <td>811e24c75b356dfee19d870838d2393f7962a8ce</td>\n",
       "      <td>2020-07-08 11:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1937976</th>\n",
       "      <td>25399942b57fcdec203000943528bf3ada78af72</td>\n",
       "      <td>2020-07-22 10:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1887283 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        7d806f8c16b04b89c5050be6fe34fe46725284c1  2020-07-16 11:00:00 UTC   \n",
       "1        9f886ba821b603f7ee8b7743b1c67e74977f6427  2020-07-05 12:10:00 UTC   \n",
       "2        f4bf9e58556aef964b509d6cb8d16e4044df48ea  2020-07-12 14:15:00 UTC   \n",
       "3        0b68615a3380664a93a5b796eb46132fa5ef699d  2020-07-10 17:50:00 UTC   \n",
       "4        3370ca7e2d9ece26c820bf1b819fadb3cd46f1b1  2020-07-08 19:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1937972  745f2411605d497ab6e2fc336f18573206e09db3  2020-07-07 16:05:00 UTC   \n",
       "1937973  f99a051f8bea5fb3bc5db59c6266a699ac680f67  2020-07-24 18:50:00 UTC   \n",
       "1937974  9c9287511ca977dfd2ae1b2811e90e9c00c66485  2020-07-14 19:00:00 UTC   \n",
       "1937975  811e24c75b356dfee19d870838d2393f7962a8ce  2020-07-08 11:15:00 UTC   \n",
       "1937976  25399942b57fcdec203000943528bf3ada78af72  2020-07-22 10:40:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          auto               795                      820   \n",
       "1           cool          hold               728                      728   \n",
       "2           cool          hold               757                      761   \n",
       "3           cool          auto               742                      752   \n",
       "4           cool          auto               779                      780   \n",
       "...          ...           ...               ...                      ...   \n",
       "1937972     cool          auto               771                      760   \n",
       "1937973     cool          auto               764                      760   \n",
       "1937974     cool          auto               765                      760   \n",
       "1937975     cool          hold               764                      760   \n",
       "1937976     cool          hold               762                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState             City  \\\n",
       "0                            733            NJ       Ocean City   \n",
       "1                            703            NJ          Bayonne   \n",
       "2                            761            NJ        Westfield   \n",
       "3                            752            NJ       Middletown   \n",
       "4                            747            NJ     Chesterfield   \n",
       "...                          ...           ...              ...   \n",
       "1937972                      760            NJ  North Brunswick   \n",
       "1937973                      760            NJ      Jersey City   \n",
       "1937974                      760            NJ           Hazlet   \n",
       "1937975                      760            NJ        Ridgewood   \n",
       "1937976                      760            NJ       Livingston   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         20             False         False            False   \n",
       "1                         10              True         False            False   \n",
       "2                          5             False         False            False   \n",
       "3                         37             False         False            False   \n",
       "4                        120             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1937972                   30             False         False            False   \n",
       "1937973                   17             False         False            False   \n",
       "1937974                   67             False         False            False   \n",
       "1937975                    0             False         False            False   \n",
       "1937976                    0              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1937972                       Gas  \n",
       "1937973                       Gas  \n",
       "1937974                       Gas  \n",
       "1937975                       Gas  \n",
       "1937976                       Gas  \n",
       "\n",
       "[1887283 rows x 14 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "jul_2020.drop(jul_2020[jul_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2020[\"Year\"] = \"2020\"\n",
    "jul_2020[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2020 = jul_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2020_ave = jul_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2020_ave.to_csv(\"data/day/NJ/jul/jul_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2021 July Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "jul_2021 = pd.read_csv(\"../data_large/NJ-day/2021-jul-day-NJ.csv\")\n",
    "\n",
    "# jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>befc73d1fb35a7d9f2c67381c15bdd0e72d1cae7</td>\n",
       "      <td>2021-07-25 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>692</td>\n",
       "      <td>655</td>\n",
       "      <td>655</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Newark</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0a6713d7f3e6fde7f01b28b16dbc8b5233afc40e</td>\n",
       "      <td>2021-07-10 18:35:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>715</td>\n",
       "      <td>715</td>\n",
       "      <td>665</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Orange</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6c7009e15f487b1a55bdb9b8c51edd3a99b8fc1d</td>\n",
       "      <td>2021-07-17 18:20:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>712</td>\n",
       "      <td>701</td>\n",
       "      <td>701</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Piscataway Township</td>\n",
       "      <td>58</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d5c59f9481aad97245fc0ae8c9e1bbb25f0962be</td>\n",
       "      <td>2021-07-20 18:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>744</td>\n",
       "      <td>744</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Neptune</td>\n",
       "      <td>15</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>dced3c100abcc7f2f98e7ceb62e6639d1ff4edee</td>\n",
       "      <td>2021-07-28 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>688</td>\n",
       "      <td>688</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Neshanic Station</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980237</th>\n",
       "      <td>6911efa295d59fc278311170dd63ec7cb9344767</td>\n",
       "      <td>2021-07-17 18:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lumberton</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980238</th>\n",
       "      <td>f18f830085ceaa87d28b51b1d7e35802033497e6</td>\n",
       "      <td>2021-07-09 17:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>60</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980239</th>\n",
       "      <td>281eb39a3c8307508e872aec94790c2673ccc3ee</td>\n",
       "      <td>2021-07-01 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>newark</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980240</th>\n",
       "      <td>adb79a920d5b3858907d6460b67c70b4c6842d71</td>\n",
       "      <td>2021-07-07 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980241</th>\n",
       "      <td>ba0791c82ba05dce044df8c5afb240523811e248</td>\n",
       "      <td>2021-07-21 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>762</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>East Brunswick</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>949529 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       befc73d1fb35a7d9f2c67381c15bdd0e72d1cae7  2021-07-25 17:25:00 UTC   \n",
       "2       0a6713d7f3e6fde7f01b28b16dbc8b5233afc40e  2021-07-10 18:35:00 UTC   \n",
       "3       6c7009e15f487b1a55bdb9b8c51edd3a99b8fc1d  2021-07-17 18:20:00 UTC   \n",
       "4       d5c59f9481aad97245fc0ae8c9e1bbb25f0962be  2021-07-20 18:50:00 UTC   \n",
       "5       dced3c100abcc7f2f98e7ceb62e6639d1ff4edee  2021-07-28 13:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "980237  6911efa295d59fc278311170dd63ec7cb9344767  2021-07-17 18:05:00 UTC   \n",
       "980238  f18f830085ceaa87d28b51b1d7e35802033497e6  2021-07-09 17:25:00 UTC   \n",
       "980239  281eb39a3c8307508e872aec94790c2673ccc3ee  2021-07-01 14:50:00 UTC   \n",
       "980240  adb79a920d5b3858907d6460b67c70b4c6842d71  2021-07-07 13:15:00 UTC   \n",
       "980241  ba0791c82ba05dce044df8c5afb240523811e248  2021-07-21 10:50:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               692                      655   \n",
       "2          auto          hold               715                      715   \n",
       "3          cool          hold               712                      701   \n",
       "4          cool          hold               762                      744   \n",
       "5          cool          hold               686                      688   \n",
       "...         ...           ...               ...                      ...   \n",
       "980237     cool          hold               762                      760   \n",
       "980238     cool          hold               763                      760   \n",
       "980239     cool          hold               760                      760   \n",
       "980240     cool          hold               759                      760   \n",
       "980241     cool          hold               762                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                           655            NJ               Newark   \n",
       "2                           665            NJ          West Orange   \n",
       "3                           701            NJ  Piscataway Township   \n",
       "4                           744            NJ              Neptune   \n",
       "5                           688            NJ     Neshanic Station   \n",
       "...                         ...           ...                  ...   \n",
       "980237                      760            NJ            Lumberton   \n",
       "980238                      760            NJ            Westfield   \n",
       "980239                      760            NJ               newark   \n",
       "980240                      760            NJ  Egg Harbor Township   \n",
       "980241                      760            NJ       East Brunswick   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         0             False         False            False   \n",
       "2                        99              True         False            False   \n",
       "3                        58              True         False            False   \n",
       "4                        15             False         False            False   \n",
       "5                        20             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "980237                    0              True         False            False   \n",
       "980238                   60             False         False            False   \n",
       "980239                    0             False         False            False   \n",
       "980240                   20             False         False            False   \n",
       "980241                    0              True         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "5                            Gas  \n",
       "...                          ...  \n",
       "980237                       Gas  \n",
       "980238                       Gas  \n",
       "980239                       Gas  \n",
       "980240                       Gas  \n",
       "980241                       Gas  \n",
       "\n",
       "[949529 rows x 14 columns]"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "jul_2021.drop(jul_2021[jul_2021['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "\n",
    "jul_2021"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "jul_2021[\"Year\"] = \"2021\"\n",
    "jul_2021[\"Month\"] = \"jul\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "jul_2021 = jul_2021.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "jul_2021_ave = jul_2021.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# jul_2021_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "jul_2021_ave.to_csv(\"data/day/NJ/jul/jul_2021_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/jul/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>794.312500</td>\n",
       "      <td>796.395833</td>\n",
       "      <td>694.729167</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>764.654731</td>\n",
       "      <td>756.971867</td>\n",
       "      <td>756.516624</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>703.583333</td>\n",
       "      <td>701.363636</td>\n",
       "      <td>651.363636</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>710.639241</td>\n",
       "      <td>709.338608</td>\n",
       "      <td>656.015823</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0129f1aa4f23beac88b2e4bd022213101f2cac8f</td>\n",
       "      <td>jul</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Haddon Township</td>\n",
       "      <td>640.850365</td>\n",
       "      <td>645.395985</td>\n",
       "      <td>649.640511</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>fe3fa3e93234275a47cc79efbfeb6989acf638c7</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Delran</td>\n",
       "      <td>719.906977</td>\n",
       "      <td>682.325581</td>\n",
       "      <td>632.325581</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1183</th>\n",
       "      <td>fe66c5051b7e13329c0c52a696a10794fa5efd7d</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Swedesboro</td>\n",
       "      <td>734.887473</td>\n",
       "      <td>728.479830</td>\n",
       "      <td>727.447983</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1184</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>797.430348</td>\n",
       "      <td>804.747512</td>\n",
       "      <td>788.419154</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1185</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>775.714286</td>\n",
       "      <td>772.319048</td>\n",
       "      <td>771.676190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1186</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>jul</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>709.310512</td>\n",
       "      <td>707.623790</td>\n",
       "      <td>686.379668</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9198 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   jul  2017     cool   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   jul  2017     cool   \n",
       "2     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   jul  2017     auto   \n",
       "3     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   jul  2017     auto   \n",
       "4     0129f1aa4f23beac88b2e4bd022213101f2cac8f   jul  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "1182  fe3fa3e93234275a47cc79efbfeb6989acf638c7   jul  2021     auto   \n",
       "1183  fe66c5051b7e13329c0c52a696a10794fa5efd7d   jul  2021     cool   \n",
       "1184  ff635b7556b71ecf409661242fa988bf1361c850   jul  2021     cool   \n",
       "1185  ffb88d949cb4d05658d8445b5cc64eb853aea426   jul  2021     cool   \n",
       "1186  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   jul  2021     auto   \n",
       "\n",
       "     CalendarEvent             City  Temperature_ctrl_ave  \\\n",
       "0             auto        Millstone            794.312500   \n",
       "1             hold        Millstone            764.654731   \n",
       "2             auto     Hillsborough            703.583333   \n",
       "3             hold     Hillsborough            710.639241   \n",
       "4             auto  Haddon Township            640.850365   \n",
       "...            ...              ...                   ...   \n",
       "1182          hold           Delran            719.906977   \n",
       "1183          hold       Swedesboro            734.887473   \n",
       "1184          hold           newark            797.430348   \n",
       "1185          hold       Livingston            775.714286   \n",
       "1186          hold          Maywood            709.310512   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      796.395833                   694.729167   \n",
       "1                      756.971867                   756.516624   \n",
       "2                      701.363636                   651.363636   \n",
       "3                      709.338608                   656.015823   \n",
       "4                      645.395985                   649.640511   \n",
       "...                           ...                          ...   \n",
       "1182                   682.325581                   632.325581   \n",
       "1183                   728.479830                   727.447983   \n",
       "1184                   804.747512                   788.419154   \n",
       "1185                   772.319048                   771.676190   \n",
       "1186                   707.623790                   686.379668   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     5.0             False         False            False  \n",
       "3                     5.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "1182                  0.0             False         False            False  \n",
       "1183                 10.0              True         False            False  \n",
       "1184                120.0             False         False            False  \n",
       "1185                  0.0             False         False            False  \n",
       "1186                 90.0             False         False            False  \n",
       "\n",
       "[9198 rows x 13 columns]"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_jul = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/jul/\" + file)\n",
    "    NJ_jul = pd.concat([NJ_jul, df])\n",
    "    \n",
    "NJ_jul"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_jul.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_jul.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## August"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2017 = pd.read_csv(\"../data_large/NJ-day/2017-aug-day-NJ.csv\")\n",
    "\n",
    "# aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b473f19c66c4eccefe0a6288ac031835e3104db9</td>\n",
       "      <td>2017-08-25 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>720</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Metuchen</td>\n",
       "      <td>37</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6c09865f6673a1f668d36eb5bfb2137545ef792f</td>\n",
       "      <td>2017-08-20 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>734</td>\n",
       "      <td>730</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mays Landing</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>87a2141d6eaebde08c70b77d8286208d2a4c1ca3</td>\n",
       "      <td>2017-08-19 15:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>716</td>\n",
       "      <td>710</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Old Bridge</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6f05ed40009c4d2311ccd097b6b6b8563a7de7c6</td>\n",
       "      <td>2017-08-23 11:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>732</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Edgewater</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e12f2bbd3ae2a7369dd8b03ffd96a5e665bb3791</td>\n",
       "      <td>2017-08-09 14:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>697</td>\n",
       "      <td>700</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hamilton Township</td>\n",
       "      <td>50</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669908</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-08-31 13:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>742</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669909</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-08-05 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>730</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669910</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-08-31 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>749</td>\n",
       "      <td>800</td>\n",
       "      <td>790</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669911</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-08-18 16:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669912</th>\n",
       "      <td>22f9a18d556c4f013209c6c834b04e5c0df714ad</td>\n",
       "      <td>2017-08-22 12:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>761</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cape May Court House</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>646738 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       b473f19c66c4eccefe0a6288ac031835e3104db9  2017-08-25 15:10:00 UTC   \n",
       "1       6c09865f6673a1f668d36eb5bfb2137545ef792f  2017-08-20 13:15:00 UTC   \n",
       "2       87a2141d6eaebde08c70b77d8286208d2a4c1ca3  2017-08-19 15:35:00 UTC   \n",
       "3       6f05ed40009c4d2311ccd097b6b6b8563a7de7c6  2017-08-23 11:35:00 UTC   \n",
       "4       e12f2bbd3ae2a7369dd8b03ffd96a5e665bb3791  2017-08-09 14:30:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "669908  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-08-31 13:50:00 UTC   \n",
       "669909  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-08-05 17:30:00 UTC   \n",
       "669910  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-08-31 15:25:00 UTC   \n",
       "669911  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-08-18 16:30:00 UTC   \n",
       "669912  22f9a18d556c4f013209c6c834b04e5c0df714ad  2017-08-22 12:15:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          cool          hold               720                      720   \n",
       "1          cool          auto               734                      730   \n",
       "2          cool          auto               716                      710   \n",
       "3          cool          hold               732                      734   \n",
       "4          cool          auto               697                      700   \n",
       "...         ...           ...               ...                      ...   \n",
       "669908     cool          hold               742                      800   \n",
       "669909     cool          auto               763                      760   \n",
       "669910     cool          hold               749                      800   \n",
       "669911     cool          hold               767                      760   \n",
       "669912     cool          auto               761                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState                  City  \\\n",
       "0                           720            NJ              Metuchen   \n",
       "1                           660            NJ          Mays Landing   \n",
       "2                           680            NJ            Old Bridge   \n",
       "3                           734            NJ             Edgewater   \n",
       "4                           700            NJ     Hamilton Township   \n",
       "...                         ...           ...                   ...   \n",
       "669908                      790            NJ  Cape May Court House   \n",
       "669909                      730            NJ  Cape May Court House   \n",
       "669910                      790            NJ  Cape May Court House   \n",
       "669911                      760            NJ  Cape May Court House   \n",
       "669912                      760            NJ  Cape May Court House   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                        37             False         False            False   \n",
       "1                        30             False         False            False   \n",
       "2                        30             False         False            False   \n",
       "3                        20             False         False             True   \n",
       "4                        50              True         False             True   \n",
       "...                     ...               ...           ...              ...   \n",
       "669908                    0             False         False            False   \n",
       "669909                    0             False         False            False   \n",
       "669910                    0             False         False            False   \n",
       "669911                    0             False         False            False   \n",
       "669912                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                       Electric  \n",
       "4                       Electric  \n",
       "...                          ...  \n",
       "669908                       Gas  \n",
       "669909                       Gas  \n",
       "669910                       Gas  \n",
       "669911                       Gas  \n",
       "669912                       Gas  \n",
       "\n",
       "[646738 rows x 14 columns]"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2017.drop(aug_2017[aug_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2017[\"Year\"] = \"2017\"\n",
    "aug_2017[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2017 = aug_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2017_ave = aug_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2017_ave.to_csv(\"data/day/NJ/aug/aug_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2018 = pd.read_csv(\"../data_large/NJ-day/2018-aug-day-NJ.csv\")\n",
    "\n",
    "# aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49dabc74be9f8342d35798fe199126b5ef58b502</td>\n",
       "      <td>2018-08-17 16:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>716</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Spring Lake</td>\n",
       "      <td>55</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9a06657297ff3a5ae7889f50724c76bd6ab0ba1b</td>\n",
       "      <td>2018-08-05 19:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>697</td>\n",
       "      <td>697</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c978fda9a1bb73a1a2376a3c821aea5fa80bfa59</td>\n",
       "      <td>2018-08-05 19:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>734</td>\n",
       "      <td>788</td>\n",
       "      <td>788</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9b9097f5da7e71defa94d636bf9b92e8f19249b6</td>\n",
       "      <td>2018-08-18 15:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>761</td>\n",
       "      <td>761</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Basking Ridge</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1cf24fdded566ea27d4330a2c02fb6d8c9bf86b7</td>\n",
       "      <td>2018-08-05 17:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>779</td>\n",
       "      <td>702</td>\n",
       "      <td>702</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Sewell</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460720</th>\n",
       "      <td>adb79a920d5b3858907d6460b67c70b4c6842d71</td>\n",
       "      <td>2018-08-04 13:30:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460721</th>\n",
       "      <td>61b3a1e8213f8e4548373b700c0a0f6c30135ba7</td>\n",
       "      <td>2018-08-03 11:45:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>763</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ridgefield</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460722</th>\n",
       "      <td>59100307e80e06a6b9d0526d75cc8cc9b7dff228</td>\n",
       "      <td>2018-08-15 11:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>751</td>\n",
       "      <td>750</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hawthorne</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460723</th>\n",
       "      <td>7858e46ce26dce8e5a31962d1953622f0178155c</td>\n",
       "      <td>2018-08-19 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Madison</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1460724</th>\n",
       "      <td>25142bea3ee05f19db5a51995044bf5325ebd357</td>\n",
       "      <td>2018-08-31 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Monroe Township</td>\n",
       "      <td>8</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1404136 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        49dabc74be9f8342d35798fe199126b5ef58b502  2018-08-17 16:45:00 UTC   \n",
       "1        9a06657297ff3a5ae7889f50724c76bd6ab0ba1b  2018-08-05 19:40:00 UTC   \n",
       "2        c978fda9a1bb73a1a2376a3c821aea5fa80bfa59  2018-08-05 19:05:00 UTC   \n",
       "3        9b9097f5da7e71defa94d636bf9b92e8f19249b6  2018-08-18 15:15:00 UTC   \n",
       "4        1cf24fdded566ea27d4330a2c02fb6d8c9bf86b7  2018-08-05 17:30:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1460720  adb79a920d5b3858907d6460b67c70b4c6842d71  2018-08-04 13:30:00 UTC   \n",
       "1460721  61b3a1e8213f8e4548373b700c0a0f6c30135ba7  2018-08-03 11:45:00 UTC   \n",
       "1460722  59100307e80e06a6b9d0526d75cc8cc9b7dff228  2018-08-15 11:05:00 UTC   \n",
       "1460723  7858e46ce26dce8e5a31962d1953622f0178155c  2018-08-19 15:10:00 UTC   \n",
       "1460724  25142bea3ee05f19db5a51995044bf5325ebd357  2018-08-31 11:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               716                      705   \n",
       "1           cool          hold               702                      697   \n",
       "2           cool          hold               734                      788   \n",
       "3           cool          hold               753                      761   \n",
       "4           cool          hold               779                      702   \n",
       "...          ...           ...               ...                      ...   \n",
       "1460720     cool          hold               764                      760   \n",
       "1460721     cool          hold               763                      760   \n",
       "1460722     cool          auto               751                      750   \n",
       "1460723     cool          hold               756                      760   \n",
       "1460724     cool          hold               764                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            705            NJ          Spring Lake   \n",
       "1                            697            NJ           Middletown   \n",
       "2                            788            NJ            Westfield   \n",
       "3                            761            NJ        Basking Ridge   \n",
       "4                            702            NJ               Sewell   \n",
       "...                          ...           ...                  ...   \n",
       "1460720                      760            NJ  Egg Harbor Township   \n",
       "1460721                      760            NJ           Ridgefield   \n",
       "1460722                      760            NJ            Hawthorne   \n",
       "1460723                      760            NJ              Madison   \n",
       "1460724                      760            NJ      Monroe Township   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         55             False         False            False   \n",
       "1                         69             False         False            False   \n",
       "2                          5             False         False            False   \n",
       "3                         30             False         False            False   \n",
       "4                         30             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1460720                   20             False         False            False   \n",
       "1460721                    0             False         False            False   \n",
       "1460722                   25             False         False            False   \n",
       "1460723                    5             False         False            False   \n",
       "1460724                    8             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1460720                       Gas  \n",
       "1460721                       Gas  \n",
       "1460722                       Gas  \n",
       "1460723                       Gas  \n",
       "1460724                       Gas  \n",
       "\n",
       "[1404136 rows x 14 columns]"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2018.drop(aug_2018[aug_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2018[\"Year\"] = \"2018\"\n",
    "aug_2018[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2018 = aug_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2018_ave = aug_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2018_ave.to_csv(\"data/day/NJ/aug/aug_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2019 = pd.read_csv(\"../data_large/NJ-day/2019-aug-day-NJ.csv\")\n",
    "\n",
    "# aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9a06657297ff3a5ae7889f50724c76bd6ab0ba1b</td>\n",
       "      <td>2019-08-12 15:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>724</td>\n",
       "      <td>723</td>\n",
       "      <td>723</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Middletown</td>\n",
       "      <td>69</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6ff5c470fde7fa363790025da01a29340c47f7f8</td>\n",
       "      <td>2019-08-05 15:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>728</td>\n",
       "      <td>723</td>\n",
       "      <td>673</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Secaucus</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b5bfc715bbebaa8507644bee6644e4582cd25bd6</td>\n",
       "      <td>2019-08-03 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>772</td>\n",
       "      <td>780</td>\n",
       "      <td>742</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mahwah</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6f05ed40009c4d2311ccd097b6b6b8563a7de7c6</td>\n",
       "      <td>2019-08-15 13:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>736</td>\n",
       "      <td>734</td>\n",
       "      <td>734</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Edgewater</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9416792730ff93925d9fdb7f1c13c618959d0381</td>\n",
       "      <td>2019-08-30 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>760</td>\n",
       "      <td>747</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cranbury Township</td>\n",
       "      <td>20</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2025177</th>\n",
       "      <td>7f4e81372e5d5d60cec5fd02a0474985135dc8e2</td>\n",
       "      <td>2019-08-15 11:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>783</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Salem</td>\n",
       "      <td>80</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2025178</th>\n",
       "      <td>8b8f1e563136f9119a1ea5b10378f8d7ebc8af18</td>\n",
       "      <td>2019-08-31 13:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>9</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2025179</th>\n",
       "      <td>d07c7da92a8a543a676bf4c3cf458f11ca8dba9e</td>\n",
       "      <td>2019-08-03 12:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fort Lee</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2025180</th>\n",
       "      <td>287f02374ce923dfd3bb89e40c480c600a9520e7</td>\n",
       "      <td>2019-08-16 12:05:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Franklin Park</td>\n",
       "      <td>118</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2025181</th>\n",
       "      <td>a1ce9693e259dd82aeac69f8aa0f5d01de9b7222</td>\n",
       "      <td>2019-08-03 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West Orange</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1952771 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        9a06657297ff3a5ae7889f50724c76bd6ab0ba1b  2019-08-12 15:10:00 UTC   \n",
       "1        6ff5c470fde7fa363790025da01a29340c47f7f8  2019-08-05 15:50:00 UTC   \n",
       "2        b5bfc715bbebaa8507644bee6644e4582cd25bd6  2019-08-03 17:40:00 UTC   \n",
       "3        6f05ed40009c4d2311ccd097b6b6b8563a7de7c6  2019-08-15 13:15:00 UTC   \n",
       "4        9416792730ff93925d9fdb7f1c13c618959d0381  2019-08-30 11:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "2025177  7f4e81372e5d5d60cec5fd02a0474985135dc8e2  2019-08-15 11:10:00 UTC   \n",
       "2025178  8b8f1e563136f9119a1ea5b10378f8d7ebc8af18  2019-08-31 13:40:00 UTC   \n",
       "2025179  d07c7da92a8a543a676bf4c3cf458f11ca8dba9e  2019-08-03 12:10:00 UTC   \n",
       "2025180  287f02374ce923dfd3bb89e40c480c600a9520e7  2019-08-16 12:05:00 UTC   \n",
       "2025181  a1ce9693e259dd82aeac69f8aa0f5d01de9b7222  2019-08-03 14:50:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               724                      723   \n",
       "1           auto          auto               728                      723   \n",
       "2           cool          hold               772                      780   \n",
       "3           cool          auto               736                      734   \n",
       "4           cool          hold               703                      760   \n",
       "...          ...           ...               ...                      ...   \n",
       "2025177     cool          auto               783                      780   \n",
       "2025178     cool          hold               757                      760   \n",
       "2025179     cool          hold               758                      760   \n",
       "2025180     cool          hold               758                      760   \n",
       "2025181     cool          auto               757                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            723            NJ         Middletown   \n",
       "1                            673            NJ           Secaucus   \n",
       "2                            742            NJ             Mahwah   \n",
       "3                            734            NJ          Edgewater   \n",
       "4                            747            NJ  Cranbury Township   \n",
       "...                          ...           ...                ...   \n",
       "2025177                      760            NJ              Salem   \n",
       "2025178                      760            NJ        Cherry Hill   \n",
       "2025179                      760            NJ           Fort Lee   \n",
       "2025180                      760            NJ      Franklin Park   \n",
       "2025181                      760            NJ        West Orange   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         69             False         False            False   \n",
       "1                         40             False         False            False   \n",
       "2                         10             False         False            False   \n",
       "3                         20             False         False             True   \n",
       "4                         20              True         False             True   \n",
       "...                      ...               ...           ...              ...   \n",
       "2025177                   80             False         False            False   \n",
       "2025178                    9             False         False            False   \n",
       "2025179                   99             False         False            False   \n",
       "2025180                  118              True         False            False   \n",
       "2025181                   25             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                        Electric  \n",
       "4                        Electric  \n",
       "...                           ...  \n",
       "2025177                       Gas  \n",
       "2025178                       Gas  \n",
       "2025179                       Gas  \n",
       "2025180                       Gas  \n",
       "2025181                       Gas  \n",
       "\n",
       "[1952771 rows x 14 columns]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2019.drop(aug_2019[aug_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2019[\"Year\"] = \"2019\"\n",
    "aug_2019[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2019 = aug_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2019_ave = aug_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2019_ave.to_csv(\"data/day/NJ/aug/aug_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 August Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "aug_2020 = pd.read_csv(\"../data_large/NJ-day/2020-aug-day-NJ.csv\")\n",
    "\n",
    "# aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>268b60939773a9951f49a06e762db62a62422056</td>\n",
       "      <td>2020-08-08 18:35:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>735</td>\n",
       "      <td>745</td>\n",
       "      <td>745</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Collegeville</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>be2df2679f5521a721caed3d48bd4a02113648c7</td>\n",
       "      <td>2020-08-23 12:25:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>775</td>\n",
       "      <td>770</td>\n",
       "      <td>608</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Basking Ridge</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9c9287511ca977dfd2ae1b2811e90e9c00c66485</td>\n",
       "      <td>2020-08-17 14:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>741</td>\n",
       "      <td>758</td>\n",
       "      <td>758</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hazlet</td>\n",
       "      <td>67</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3584d50be0c0ac7fa814da32446fca6c212a6aba</td>\n",
       "      <td>2020-08-10 19:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>709</td>\n",
       "      <td>695</td>\n",
       "      <td>645</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montclair</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cbc8294ee740f98c64db35add9bee5032c645508</td>\n",
       "      <td>2020-08-09 14:10:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>750</td>\n",
       "      <td>750</td>\n",
       "      <td>749</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Colonia</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1891229</th>\n",
       "      <td>e5f0583c0c069406881c13d9a6bc19a4110cbc78</td>\n",
       "      <td>2020-08-02 17:40:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>757</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cranford</td>\n",
       "      <td>99</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1891230</th>\n",
       "      <td>bb203770e52355aa59ebf5879a6237adc572ccc7</td>\n",
       "      <td>2020-08-30 10:50:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Matawan</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1891231</th>\n",
       "      <td>1400685c74fb4010dd2a97b109c51150e92017a6</td>\n",
       "      <td>2020-08-29 15:25:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>764</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ewing</td>\n",
       "      <td>60</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1891232</th>\n",
       "      <td>adb79a920d5b3858907d6460b67c70b4c6842d71</td>\n",
       "      <td>2020-08-18 18:55:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>759</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1891233</th>\n",
       "      <td>d8f2f9944a3162ee9153d9ad039535cd7dbafeaf</td>\n",
       "      <td>2020-08-18 19:15:00 UTC</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>779</td>\n",
       "      <td>780</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lacey Township</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1827170 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        268b60939773a9951f49a06e762db62a62422056  2020-08-08 18:35:00 UTC   \n",
       "1        be2df2679f5521a721caed3d48bd4a02113648c7  2020-08-23 12:25:00 UTC   \n",
       "2        9c9287511ca977dfd2ae1b2811e90e9c00c66485  2020-08-17 14:50:00 UTC   \n",
       "3        3584d50be0c0ac7fa814da32446fca6c212a6aba  2020-08-10 19:50:00 UTC   \n",
       "4        cbc8294ee740f98c64db35add9bee5032c645508  2020-08-09 14:10:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1891229  e5f0583c0c069406881c13d9a6bc19a4110cbc78  2020-08-02 17:40:00 UTC   \n",
       "1891230  bb203770e52355aa59ebf5879a6237adc572ccc7  2020-08-30 10:50:00 UTC   \n",
       "1891231  1400685c74fb4010dd2a97b109c51150e92017a6  2020-08-29 15:25:00 UTC   \n",
       "1891232  adb79a920d5b3858907d6460b67c70b4c6842d71  2020-08-18 18:55:00 UTC   \n",
       "1891233  d8f2f9944a3162ee9153d9ad039535cd7dbafeaf  2020-08-18 19:15:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           cool          hold               735                      745   \n",
       "1           auto          auto               775                      770   \n",
       "2           cool          auto               741                      758   \n",
       "3           auto          hold               709                      695   \n",
       "4           cool          hold               750                      750   \n",
       "...          ...           ...               ...                      ...   \n",
       "1891229     cool          auto               757                      760   \n",
       "1891230     cool          hold               755                      760   \n",
       "1891231     cool          hold               764                      760   \n",
       "1891232     cool          hold               759                      760   \n",
       "1891233     cool          auto               779                      780   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState                 City  \\\n",
       "0                            745            NJ         Collegeville   \n",
       "1                            608            NJ        Basking Ridge   \n",
       "2                            758            NJ               Hazlet   \n",
       "3                            645            NJ            Montclair   \n",
       "4                            749            NJ              Colonia   \n",
       "...                          ...           ...                  ...   \n",
       "1891229                      760            NJ             Cranford   \n",
       "1891230                      760            NJ              Matawan   \n",
       "1891231                      760            NJ                Ewing   \n",
       "1891232                      760            NJ  Egg Harbor Township   \n",
       "1891233                      760            NJ       Lacey Township   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0             False         False            False   \n",
       "1                         25             False         False            False   \n",
       "2                         67             False         False            False   \n",
       "3                          0             False         False            False   \n",
       "4                          0             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1891229                   99              True         False            False   \n",
       "1891230                   60              True         False             True   \n",
       "1891231                   60              True         False            False   \n",
       "1891232                   20             False         False            False   \n",
       "1891233                    0              True         False             True   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1891229                       Gas  \n",
       "1891230                  Electric  \n",
       "1891231                       Gas  \n",
       "1891232                       Gas  \n",
       "1891233                  Electric  \n",
       "\n",
       "[1827170 rows x 14 columns]"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "aug_2020.drop(aug_2020[aug_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "aug_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "aug_2020[\"Year\"] = \"2020\"\n",
    "aug_2020[\"Month\"] = \"aug\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "aug_2020 = aug_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "aug_2020_ave = aug_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# aug_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "aug_2020_ave.to_csv(\"data/day/NJ/aug/aug_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": true
   },
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/aug/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>775.569975</td>\n",
       "      <td>774.788804</td>\n",
       "      <td>668.452926</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>773.368932</td>\n",
       "      <td>767.621359</td>\n",
       "      <td>767.317152</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>709.096579</td>\n",
       "      <td>707.939638</td>\n",
       "      <td>657.931590</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>675.947368</td>\n",
       "      <td>675.410526</td>\n",
       "      <td>624.842105</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0129f1aa4f23beac88b2e4bd022213101f2cac8f</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Haddon Township</td>\n",
       "      <td>619.096234</td>\n",
       "      <td>638.418410</td>\n",
       "      <td>625.585774</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2279</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>779.097668</td>\n",
       "      <td>770.459184</td>\n",
       "      <td>769.724490</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2280</th>\n",
       "      <td>ff6e08f27d804c684e9ea9002f77db7e6f597cc3</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Howell</td>\n",
       "      <td>731.355932</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>15.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2281</th>\n",
       "      <td>ffacf5dda3b0257b3891ad670f2d1f42187e2277</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>721.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2282</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>783.851852</td>\n",
       "      <td>780.148148</td>\n",
       "      <td>779.666667</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2283</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>aug</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>723.790960</td>\n",
       "      <td>722.030132</td>\n",
       "      <td>680.000000</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>7612 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   aug  2017     cool   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   aug  2017     cool   \n",
       "2     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   aug  2017     auto   \n",
       "3     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   aug  2017     auto   \n",
       "4     0129f1aa4f23beac88b2e4bd022213101f2cac8f   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2279  ff635b7556b71ecf409661242fa988bf1361c850   aug  2020     cool   \n",
       "2280  ff6e08f27d804c684e9ea9002f77db7e6f597cc3   aug  2020     auto   \n",
       "2281  ffacf5dda3b0257b3891ad670f2d1f42187e2277   aug  2020     heat   \n",
       "2282  ffb88d949cb4d05658d8445b5cc64eb853aea426   aug  2020     cool   \n",
       "2283  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   aug  2020     auto   \n",
       "\n",
       "     CalendarEvent             City  Temperature_ctrl_ave  \\\n",
       "0             auto        Millstone            775.569975   \n",
       "1             hold        Millstone            773.368932   \n",
       "2             auto     Hillsborough            709.096579   \n",
       "3             hold     Hillsborough            675.947368   \n",
       "4             auto  Haddon Township            619.096234   \n",
       "...            ...              ...                   ...   \n",
       "2279          hold           newark            779.097668   \n",
       "2280          auto           Howell            731.355932   \n",
       "2281          auto        Oak Ridge            721.000000   \n",
       "2282          hold       Livingston            783.851852   \n",
       "2283          hold          Maywood            723.790960   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      774.788804                   668.452926   \n",
       "1                      767.621359                   767.317152   \n",
       "2                      707.939638                   657.931590   \n",
       "3                      675.410526                   624.842105   \n",
       "4                      638.418410                   625.585774   \n",
       "...                           ...                          ...   \n",
       "2279                   770.459184                   769.724490   \n",
       "2280                   720.000000                   670.000000   \n",
       "2281                   750.000000                   750.000000   \n",
       "2282                   780.148148                   779.666667   \n",
       "2283                   722.030132                   680.000000   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     5.0             False         False            False  \n",
       "3                     5.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2279                120.0             False         False            False  \n",
       "2280                 15.0             False         False            False  \n",
       "2281                 99.0              True         False            False  \n",
       "2282                  0.0             False         False            False  \n",
       "2283                 90.0             False         False            False  \n",
       "\n",
       "[7612 rows x 13 columns]"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_aug = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/aug/\" + file)\n",
    "    NJ_aug = pd.concat([NJ_aug, df])\n",
    "    \n",
    "NJ_aug"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_aug.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_aug.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## September"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## October"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## November"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "## December"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2017 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2017 = pd.read_csv(\"../data_large/NJ-day/2017-dec-day-NJ.csv\")\n",
    "\n",
    "# dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c978fda9a1bb73a1a2376a3c821aea5fa80bfa59</td>\n",
       "      <td>2017-12-01 16:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>683</td>\n",
       "      <td>653</td>\n",
       "      <td>653</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Westfield</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>de606ee4b2c3dcb4d48c7be11f59038350cc1e7c</td>\n",
       "      <td>2017-12-04 12:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>678</td>\n",
       "      <td>810</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>South River</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>de606ee4b2c3dcb4d48c7be11f59038350cc1e7c</td>\n",
       "      <td>2017-12-24 12:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>675</td>\n",
       "      <td>810</td>\n",
       "      <td>640</td>\n",
       "      <td>NJ</td>\n",
       "      <td>South River</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44a932e8bd5eb3becee2579895587f54a33b66fd</td>\n",
       "      <td>2017-12-03 11:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>668</td>\n",
       "      <td>684</td>\n",
       "      <td>684</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Leonardo</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2a01d81b8e560607d42f278ba5234d19e8d3740e</td>\n",
       "      <td>2017-12-04 15:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>704</td>\n",
       "      <td>755</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lawrence</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959207</th>\n",
       "      <td>69933dd9a30b29ff8606fe3a3e78f1baef8eb714</td>\n",
       "      <td>2017-12-24 16:10:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Clifton</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959208</th>\n",
       "      <td>0a41e735b50199a402317d745aa10f29020ee3c3</td>\n",
       "      <td>2017-12-12 11:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>733</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Elmer</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959209</th>\n",
       "      <td>ba36eb1f67ff36cde03a2dc11fd8b75c64b3f722</td>\n",
       "      <td>2017-12-08 14:20:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>686</td>\n",
       "      <td>760</td>\n",
       "      <td>690</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Weehawken</td>\n",
       "      <td>107</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959210</th>\n",
       "      <td>7658acde76a881ac0f31b56dfe570f483d84c835</td>\n",
       "      <td>2017-12-24 13:45:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Colts Neck</td>\n",
       "      <td>50</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959211</th>\n",
       "      <td>ac2c56c04414c26828a926ce098ca769d59573b8</td>\n",
       "      <td>2017-12-05 17:40:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Woodbridge</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>908420 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      Identifier                date_time  \\\n",
       "0       c978fda9a1bb73a1a2376a3c821aea5fa80bfa59  2017-12-01 16:05:00 UTC   \n",
       "1       de606ee4b2c3dcb4d48c7be11f59038350cc1e7c  2017-12-04 12:10:00 UTC   \n",
       "2       de606ee4b2c3dcb4d48c7be11f59038350cc1e7c  2017-12-24 12:05:00 UTC   \n",
       "3       44a932e8bd5eb3becee2579895587f54a33b66fd  2017-12-03 11:40:00 UTC   \n",
       "4       2a01d81b8e560607d42f278ba5234d19e8d3740e  2017-12-04 15:40:00 UTC   \n",
       "...                                          ...                      ...   \n",
       "959207  69933dd9a30b29ff8606fe3a3e78f1baef8eb714  2017-12-24 16:10:00 UTC   \n",
       "959208  0a41e735b50199a402317d745aa10f29020ee3c3  2017-12-12 11:15:00 UTC   \n",
       "959209  ba36eb1f67ff36cde03a2dc11fd8b75c64b3f722  2017-12-08 14:20:00 UTC   \n",
       "959210  7658acde76a881ac0f31b56dfe570f483d84c835  2017-12-24 13:45:00 UTC   \n",
       "959211  ac2c56c04414c26828a926ce098ca769d59573b8  2017-12-05 17:40:00 UTC   \n",
       "\n",
       "       HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0          heat          hold               683                      653   \n",
       "1          heat          auto               678                      810   \n",
       "2          heat          auto               675                      810   \n",
       "3          heat          hold               668                      684   \n",
       "4          auto          hold               704                      755   \n",
       "...         ...           ...               ...                      ...   \n",
       "959207     heat          hold               755                      760   \n",
       "959208     heat          auto               733                      760   \n",
       "959209     auto          hold               686                      760   \n",
       "959210     auto          hold               701                      760   \n",
       "959211     auto          hold               698                      760   \n",
       "\n",
       "        TemperatureExpectedHeat ProvinceState         City  \\\n",
       "0                           653            NJ    Westfield   \n",
       "1                           680            NJ  South River   \n",
       "2                           640            NJ  South River   \n",
       "3                           684            NJ     Leonardo   \n",
       "4                           705            NJ     Lawrence   \n",
       "...                         ...           ...          ...   \n",
       "959207                      760            NJ      Clifton   \n",
       "959208                      760            NJ        Elmer   \n",
       "959209                      690            NJ    Weehawken   \n",
       "959210                      700            NJ   Colts Neck   \n",
       "959211                      700            NJ   Woodbridge   \n",
       "\n",
       "        Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         5             False         False            False   \n",
       "1                         5             False         False            False   \n",
       "2                         5             False         False            False   \n",
       "3                         5             False         False            False   \n",
       "4                        10             False         False            False   \n",
       "...                     ...               ...           ...              ...   \n",
       "959207                    5             False         False            False   \n",
       "959208                   25             False         False            False   \n",
       "959209                  107             False         False            False   \n",
       "959210                   50             False         False            False   \n",
       "959211                    0             False         False            False   \n",
       "\n",
       "       Auxilliary_Heat_Fuel_Type  \n",
       "0                            Gas  \n",
       "1                            Gas  \n",
       "2                            Gas  \n",
       "3                            Gas  \n",
       "4                            Gas  \n",
       "...                          ...  \n",
       "959207                       Gas  \n",
       "959208                       Gas  \n",
       "959209                       Gas  \n",
       "959210                       Gas  \n",
       "959211                       Gas  \n",
       "\n",
       "[908420 rows x 14 columns]"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2017.drop(dec_2017[dec_2017['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2017[\"Year\"] = \"2017\"\n",
    "dec_2017[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2017 = dec_2017.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2017_ave = dec_2017.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2017_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2017_ave.to_csv(\"data/day/NJ/dec/dec_2017_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2018 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2018 = pd.read_csv(\"../data_large/NJ-day/2018-dec-day-NJ.csv\")\n",
    "\n",
    "# dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>298d090c07f3843a9e9ea44a3dac1e7fa53206c6</td>\n",
       "      <td>2018-12-09 18:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>760</td>\n",
       "      <td>743</td>\n",
       "      <td>743</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West New York</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>be78777594a54b99d136c1b41213a62ed009c45c</td>\n",
       "      <td>2018-12-02 17:25:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>701</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Morris</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70ad65a35913e0bb2ea598e92dbd5f3996c88ec5</td>\n",
       "      <td>2018-12-29 19:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>681</td>\n",
       "      <td>685</td>\n",
       "      <td>685</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Montgomery</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d48bc977fa2820c8ebdb5379d3b296ee47cf7e3b</td>\n",
       "      <td>2018-12-19 17:45:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>703</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>East Brunswick</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d48bc977fa2820c8ebdb5379d3b296ee47cf7e3b</td>\n",
       "      <td>2018-12-27 13:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>676</td>\n",
       "      <td>703</td>\n",
       "      <td>680</td>\n",
       "      <td>NJ</td>\n",
       "      <td>East Brunswick</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552290</th>\n",
       "      <td>ba750e1e8ea0125697626086bdd9a980b401746a</td>\n",
       "      <td>2018-12-02 15:15:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>767</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Berlin</td>\n",
       "      <td>17</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552291</th>\n",
       "      <td>644c49d19f4bda542983482244b1121fc3be036c</td>\n",
       "      <td>2018-12-17 18:05:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>756</td>\n",
       "      <td>760</td>\n",
       "      <td>750</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Linden</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552292</th>\n",
       "      <td>d07c7da92a8a543a676bf4c3cf458f11ca8dba9e</td>\n",
       "      <td>2018-12-14 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>758</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Fort Lee</td>\n",
       "      <td>99</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552293</th>\n",
       "      <td>2bb3929a2e7a73f199948464d2a4f85442ddb0ca</td>\n",
       "      <td>2018-12-25 18:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>681</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Trenton</td>\n",
       "      <td>10</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1552294</th>\n",
       "      <td>e5e6d0a7916e8f578600adc73d4837644b417c48</td>\n",
       "      <td>2018-12-06 14:10:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Morris</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1461912 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        298d090c07f3843a9e9ea44a3dac1e7fa53206c6  2018-12-09 18:20:00 UTC   \n",
       "1        be78777594a54b99d136c1b41213a62ed009c45c  2018-12-02 17:25:00 UTC   \n",
       "2        70ad65a35913e0bb2ea598e92dbd5f3996c88ec5  2018-12-29 19:50:00 UTC   \n",
       "3        d48bc977fa2820c8ebdb5379d3b296ee47cf7e3b  2018-12-19 17:45:00 UTC   \n",
       "4        d48bc977fa2820c8ebdb5379d3b296ee47cf7e3b  2018-12-27 13:55:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1552290  ba750e1e8ea0125697626086bdd9a980b401746a  2018-12-02 15:15:00 UTC   \n",
       "1552291  644c49d19f4bda542983482244b1121fc3be036c  2018-12-17 18:05:00 UTC   \n",
       "1552292  d07c7da92a8a543a676bf4c3cf458f11ca8dba9e  2018-12-14 14:20:00 UTC   \n",
       "1552293  2bb3929a2e7a73f199948464d2a4f85442ddb0ca  2018-12-25 18:30:00 UTC   \n",
       "1552294  e5e6d0a7916e8f578600adc73d4837644b417c48  2018-12-06 14:10:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               760                      743   \n",
       "1           heat          hold               701                      705   \n",
       "2           heat          hold               681                      685   \n",
       "3           heat          auto               681                      703   \n",
       "4           heat          hold               676                      703   \n",
       "...          ...           ...               ...                      ...   \n",
       "1552290     heat          hold               767                      760   \n",
       "1552291     heat          auto               756                      760   \n",
       "1552292     heat          hold               758                      760   \n",
       "1552293     heat          auto               681                      760   \n",
       "1552294     auto          hold               706                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState            City  \\\n",
       "0                            743            NJ   West New York   \n",
       "1                            705            NJ          Morris   \n",
       "2                            685            NJ      Montgomery   \n",
       "3                            680            NJ  East Brunswick   \n",
       "4                            680            NJ  East Brunswick   \n",
       "...                          ...           ...             ...   \n",
       "1552290                      760            NJ          Berlin   \n",
       "1552291                      750            NJ          Linden   \n",
       "1552292                      760            NJ        Fort Lee   \n",
       "1552293                      660            NJ         Trenton   \n",
       "1552294                      710            NJ          Morris   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False             True   \n",
       "1                         20             False         False            False   \n",
       "2                         40             False         False            False   \n",
       "3                          0              True         False            False   \n",
       "4                          0              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1552290                   17             False         False            False   \n",
       "1552291                    0             False         False            False   \n",
       "1552292                   99             False         False            False   \n",
       "1552293                   10             False         False            False   \n",
       "1552294                    0             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                        Electric  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1552290                       Gas  \n",
       "1552291                       Gas  \n",
       "1552292                       Gas  \n",
       "1552293                       Gas  \n",
       "1552294                       Gas  \n",
       "\n",
       "[1461912 rows x 14 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2018.drop(dec_2018[dec_2018['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2018"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2018[\"Year\"] = \"2018\"\n",
    "dec_2018[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2018 = dec_2018.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2018_ave = dec_2018.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2018_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2018_ave.to_csv(\"data/day/NJ/dec/dec_2018_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2019 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2019 = pd.read_csv(\"../data_large/NJ-day/2019-dec-day-NJ.csv\")\n",
    "\n",
    "# dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>453ba6643107e17bd5d4d10d6a8d36c6e5e06ccb</td>\n",
       "      <td>2019-12-13 16:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>703</td>\n",
       "      <td>725</td>\n",
       "      <td>675</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Hoboken</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c6b6bc6efa103ff2ad9b7cecd1ded574f8bf4bd3</td>\n",
       "      <td>2019-12-10 15:55:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>707</td>\n",
       "      <td>689</td>\n",
       "      <td>689</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Wall</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>71cd3bad8e9c69534d8339e0f2efe6eb2bda909e</td>\n",
       "      <td>2019-12-18 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>645</td>\n",
       "      <td>652</td>\n",
       "      <td>652</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lafayette</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71cd3bad8e9c69534d8339e0f2efe6eb2bda909e</td>\n",
       "      <td>2019-12-22 12:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>661</td>\n",
       "      <td>662</td>\n",
       "      <td>662</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Lafayette</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>9c3324feb0c5b373e621d724b95f19af2010abaa</td>\n",
       "      <td>2019-12-15 15:00:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>726</td>\n",
       "      <td>729</td>\n",
       "      <td>729</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mahwah</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951286</th>\n",
       "      <td>644c49d19f4bda542983482244b1121fc3be036c</td>\n",
       "      <td>2019-12-07 12:40:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>694</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Linden</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951287</th>\n",
       "      <td>599ee305b52c541b948c1d82c2aeef4946d817b5</td>\n",
       "      <td>2019-12-04 14:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>619</td>\n",
       "      <td>760</td>\n",
       "      <td>620</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Voorhees</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951288</th>\n",
       "      <td>7e2f3c9de5fb143793f233c07a049710196bc56a</td>\n",
       "      <td>2019-12-24 13:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>706</td>\n",
       "      <td>760</td>\n",
       "      <td>710</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Cherry Hill</td>\n",
       "      <td>100</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951289</th>\n",
       "      <td>653b5617a4575acb4aafa5889cb3463c1298b9bf</td>\n",
       "      <td>2019-12-20 17:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Plainfield</td>\n",
       "      <td>40</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1951290</th>\n",
       "      <td>a38ae5396879c4b00d7614b0b67a69cd2458470a</td>\n",
       "      <td>2019-12-20 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>755</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Sicklerville</td>\n",
       "      <td>19</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1826463 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        453ba6643107e17bd5d4d10d6a8d36c6e5e06ccb  2019-12-13 16:55:00 UTC   \n",
       "1        c6b6bc6efa103ff2ad9b7cecd1ded574f8bf4bd3  2019-12-10 15:55:00 UTC   \n",
       "2        71cd3bad8e9c69534d8339e0f2efe6eb2bda909e  2019-12-18 12:40:00 UTC   \n",
       "3        71cd3bad8e9c69534d8339e0f2efe6eb2bda909e  2019-12-22 12:20:00 UTC   \n",
       "4        9c3324feb0c5b373e621d724b95f19af2010abaa  2019-12-15 15:00:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1951286  644c49d19f4bda542983482244b1121fc3be036c  2019-12-07 12:40:00 UTC   \n",
       "1951287  599ee305b52c541b948c1d82c2aeef4946d817b5  2019-12-04 14:50:00 UTC   \n",
       "1951288  7e2f3c9de5fb143793f233c07a049710196bc56a  2019-12-24 13:50:00 UTC   \n",
       "1951289  653b5617a4575acb4aafa5889cb3463c1298b9bf  2019-12-20 17:50:00 UTC   \n",
       "1951290  a38ae5396879c4b00d7614b0b67a69cd2458470a  2019-12-20 17:30:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           heat          hold               703                      725   \n",
       "1           heat          hold               707                      689   \n",
       "2           heat          hold               645                      652   \n",
       "3           heat          hold               661                      662   \n",
       "4           heat          hold               726                      729   \n",
       "...          ...           ...               ...                      ...   \n",
       "1951286     heat          hold               694                      760   \n",
       "1951287     auto          hold               619                      760   \n",
       "1951288     auto          auto               706                      760   \n",
       "1951289     heat          hold               755                      760   \n",
       "1951290     heat          auto               755                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState              City  \\\n",
       "0                            675            NJ           Hoboken   \n",
       "1                            689            NJ              Wall   \n",
       "2                            652            NJ         Lafayette   \n",
       "3                            662            NJ         Lafayette   \n",
       "4                            729            NJ            Mahwah   \n",
       "...                          ...           ...               ...   \n",
       "1951286                      700            NJ            Linden   \n",
       "1951287                      620            NJ          Voorhees   \n",
       "1951288                      710            NJ       Cherry Hill   \n",
       "1951289                      760            NJ  North Plainfield   \n",
       "1951290                      760            NJ      Sicklerville   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                          0              True         False            False   \n",
       "1                          0             False         False            False   \n",
       "2                          0             False         False            False   \n",
       "3                          0             False         False            False   \n",
       "4                         20             False         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1951286                    0             False         False            False   \n",
       "1951287                   30              True         False            False   \n",
       "1951288                  100             False         False            False   \n",
       "1951289                   40             False         False            False   \n",
       "1951290                   19              True         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                             Gas  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1951286                       Gas  \n",
       "1951287                       Gas  \n",
       "1951288                       Gas  \n",
       "1951289                       Gas  \n",
       "1951290                       Gas  \n",
       "\n",
       "[1826463 rows x 14 columns]"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2019.drop(dec_2019[dec_2019['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2019"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2019[\"Year\"] = \"2019\"\n",
    "dec_2019[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2019 = dec_2019.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2019_ave = dec_2019.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2019_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2019_ave.to_csv(\"data/day/NJ/dec/dec_2019_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2020 December Day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read in month csv for state\n",
    "dec_2020 = pd.read_csv(\"../data_large/NJ-day/2020-dec-day-NJ.csv\")\n",
    "\n",
    "# dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>date_time</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>Temperature_ctrl</th>\n",
       "      <th>TemperatureExpectedCool</th>\n",
       "      <th>TemperatureExpectedHeat</th>\n",
       "      <th>ProvinceState</th>\n",
       "      <th>City</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "      <th>Auxilliary_Heat_Fuel_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>193be8b5b1d1c343b24f78aa1adc3f719c7d6484</td>\n",
       "      <td>2020-12-18 17:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>660</td>\n",
       "      <td>830</td>\n",
       "      <td>650</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Scotch Plains</td>\n",
       "      <td>70</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>eea162e43c8dd1f3950760333c8b73c940980e97</td>\n",
       "      <td>2020-12-29 11:50:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>711</td>\n",
       "      <td>815</td>\n",
       "      <td>716</td>\n",
       "      <td>NJ</td>\n",
       "      <td>West New York</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>Electric</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d37109679aedec1fce5a81b6cc4ed9a7278e7c05</td>\n",
       "      <td>2020-12-03 17:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>697</td>\n",
       "      <td>699</td>\n",
       "      <td>699</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Jackson</td>\n",
       "      <td>30</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7720ec22884af3da9000a9b7a6a6f4ac67a7b10e</td>\n",
       "      <td>2020-12-12 14:30:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>710</td>\n",
       "      <td>716</td>\n",
       "      <td>716</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Mount Laurel</td>\n",
       "      <td>20</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d7d4bbf4c40cafc1d3f46c2e67429754c388951c</td>\n",
       "      <td>2020-12-20 14:50:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>702</td>\n",
       "      <td>705</td>\n",
       "      <td>705</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Ventnor City</td>\n",
       "      <td>25</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567019</th>\n",
       "      <td>7ea729c651850cfbc825e2ddbf8a15cf91b6d042</td>\n",
       "      <td>2020-12-20 14:20:00 UTC</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>753</td>\n",
       "      <td>760</td>\n",
       "      <td>760</td>\n",
       "      <td>NJ</td>\n",
       "      <td>East Rutherford</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567020</th>\n",
       "      <td>acf49d5329268c9aee85ab01e38e1bb05277d876</td>\n",
       "      <td>2020-12-18 18:05:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>698</td>\n",
       "      <td>760</td>\n",
       "      <td>700</td>\n",
       "      <td>NJ</td>\n",
       "      <td>North Brunswick</td>\n",
       "      <td>30</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567021</th>\n",
       "      <td>1a64c8a947516a7687bc6a237802da436b21c7c4</td>\n",
       "      <td>2020-12-14 19:15:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>665</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Howell</td>\n",
       "      <td>0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567022</th>\n",
       "      <td>2406f1c08f3758a93961d01674e31daf625fab21</td>\n",
       "      <td>2020-12-03 16:55:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>666</td>\n",
       "      <td>760</td>\n",
       "      <td>670</td>\n",
       "      <td>NJ</td>\n",
       "      <td>Madison</td>\n",
       "      <td>90</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1567023</th>\n",
       "      <td>88cd9c654c1dfe899d5a8f41fd40cb93fa9dee99</td>\n",
       "      <td>2020-12-05 16:00:00 UTC</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>652</td>\n",
       "      <td>760</td>\n",
       "      <td>660</td>\n",
       "      <td>NJ</td>\n",
       "      <td>South Bound Brook</td>\n",
       "      <td>35</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Gas</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1480699 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Identifier                date_time  \\\n",
       "0        193be8b5b1d1c343b24f78aa1adc3f719c7d6484  2020-12-18 17:00:00 UTC   \n",
       "1        eea162e43c8dd1f3950760333c8b73c940980e97  2020-12-29 11:50:00 UTC   \n",
       "2        d37109679aedec1fce5a81b6cc4ed9a7278e7c05  2020-12-03 17:30:00 UTC   \n",
       "3        7720ec22884af3da9000a9b7a6a6f4ac67a7b10e  2020-12-12 14:30:00 UTC   \n",
       "4        d7d4bbf4c40cafc1d3f46c2e67429754c388951c  2020-12-20 14:50:00 UTC   \n",
       "...                                           ...                      ...   \n",
       "1567019  7ea729c651850cfbc825e2ddbf8a15cf91b6d042  2020-12-20 14:20:00 UTC   \n",
       "1567020  acf49d5329268c9aee85ab01e38e1bb05277d876  2020-12-18 18:05:00 UTC   \n",
       "1567021  1a64c8a947516a7687bc6a237802da436b21c7c4  2020-12-14 19:15:00 UTC   \n",
       "1567022  2406f1c08f3758a93961d01674e31daf625fab21  2020-12-03 16:55:00 UTC   \n",
       "1567023  88cd9c654c1dfe899d5a8f41fd40cb93fa9dee99  2020-12-05 16:00:00 UTC   \n",
       "\n",
       "        HvacMode CalendarEvent  Temperature_ctrl  TemperatureExpectedCool  \\\n",
       "0           auto          hold               660                      830   \n",
       "1           auto          hold               711                      815   \n",
       "2           heat          hold               697                      699   \n",
       "3           heat          hold               710                      716   \n",
       "4           heat          hold               702                      705   \n",
       "...          ...           ...               ...                      ...   \n",
       "1567019     heat          hold               753                      760   \n",
       "1567020     auto          auto               698                      760   \n",
       "1567021     auto          hold               665                      760   \n",
       "1567022     auto          hold               666                      760   \n",
       "1567023     auto          hold               652                      760   \n",
       "\n",
       "         TemperatureExpectedHeat ProvinceState               City  \\\n",
       "0                            650            NJ      Scotch Plains   \n",
       "1                            716            NJ      West New York   \n",
       "2                            699            NJ            Jackson   \n",
       "3                            716            NJ       Mount Laurel   \n",
       "4                            705            NJ       Ventnor City   \n",
       "...                          ...           ...                ...   \n",
       "1567019                      760            NJ    East Rutherford   \n",
       "1567020                      700            NJ    North Brunswick   \n",
       "1567021                      660            NJ             Howell   \n",
       "1567022                      670            NJ            Madison   \n",
       "1567023                      660            NJ  South Bound Brook   \n",
       "\n",
       "         Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \\\n",
       "0                         70             False         False            False   \n",
       "1                          0              True         False             True   \n",
       "2                         30             False         False            False   \n",
       "3                         20             False         False            False   \n",
       "4                         25              True         False            False   \n",
       "...                      ...               ...           ...              ...   \n",
       "1567019                    0             False         False            False   \n",
       "1567020                   30              True         False            False   \n",
       "1567021                    0              True         False            False   \n",
       "1567022                   90             False         False            False   \n",
       "1567023                   35             False         False            False   \n",
       "\n",
       "        Auxilliary_Heat_Fuel_Type  \n",
       "0                             Gas  \n",
       "1                        Electric  \n",
       "2                             Gas  \n",
       "3                             Gas  \n",
       "4                             Gas  \n",
       "...                           ...  \n",
       "1567019                       Gas  \n",
       "1567020                       Gas  \n",
       "1567021                       Gas  \n",
       "1567022                       Gas  \n",
       "1567023                       Gas  \n",
       "\n",
       "[1480699 rows x 14 columns]"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove predetermined outliers before aggregating.\n",
    "# TemperatureExpectedCool >= 850\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] >= 850].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] >= 850].index, inplace = True)\n",
    "\n",
    "# TemperatureExpectedHeat <= 600\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedCool'] <= 600].index, inplace = True)\n",
    "dec_2020.drop(dec_2020[dec_2020['TemperatureExpectedHeat'] <= 600].index, inplace = True)\n",
    "\n",
    "dec_2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Add year and month\n",
    "\n",
    "dec_2020[\"Year\"] = \"2020\"\n",
    "dec_2020[\"Month\"] = \"dec\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns to label the aggregates\n",
    "\n",
    "dec_2020 = dec_2020.rename(columns={\"Temperature_ctrl\":\"Temperature_ctrl_ave\", \n",
    "                         \"TemperatureExpectedCool\":\"TemperatureExpectedCool_ave\", \n",
    "                         \"TemperatureExpectedHeat\":\"TemperatureExpectedHeat_ave\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dec_2020_ave = dec_2020.groupby(['Identifier', 'Month', 'Year','HvacMode','CalendarEvent', 'City']).mean()\n",
    "\n",
    "# dec_2020_ave"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export CSV file\n",
    "\n",
    "dec_2020_ave.to_csv(\"data/day/NJ/dec/dec_2020_ave.csv\", header=True, index=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine month CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"data/day/NJ/dec/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>674.580038</td>\n",
       "      <td>674.732580</td>\n",
       "      <td>673.749529</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>695.525301</td>\n",
       "      <td>700.939759</td>\n",
       "      <td>700.679518</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0077dbe1c8326d5fbe2f7379172ae07159a24393</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>630.330435</td>\n",
       "      <td>650.268478</td>\n",
       "      <td>630.450000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0077dbe1c8326d5fbe2f7379172ae07159a24393</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Ridgewood</td>\n",
       "      <td>653.267806</td>\n",
       "      <td>661.336182</td>\n",
       "      <td>638.943020</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00cfe51c25b825270bc4509dfadca8d4f50a4e99</td>\n",
       "      <td>dec</td>\n",
       "      <td>2017</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Medford</td>\n",
       "      <td>620.880000</td>\n",
       "      <td>651.400000</td>\n",
       "      <td>620.280000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2300</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>newark</td>\n",
       "      <td>680.130178</td>\n",
       "      <td>692.143984</td>\n",
       "      <td>688.765286</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2301</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>684.165545</td>\n",
       "      <td>689.464334</td>\n",
       "      <td>685.562584</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2302</th>\n",
       "      <td>ffacf5dda3b0257b3891ad670f2d1f42187e2277</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>auto</td>\n",
       "      <td>Oak Ridge</td>\n",
       "      <td>732.026316</td>\n",
       "      <td>727.631579</td>\n",
       "      <td>724.500000</td>\n",
       "      <td>99.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2303</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>heat</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>713.411765</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>720.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2304</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>dec</td>\n",
       "      <td>2020</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>705.243827</td>\n",
       "      <td>739.351852</td>\n",
       "      <td>708.299383</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8669 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   dec  2017     heat   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   dec  2017     heat   \n",
       "2     0077dbe1c8326d5fbe2f7379172ae07159a24393   dec  2017     heat   \n",
       "3     0077dbe1c8326d5fbe2f7379172ae07159a24393   dec  2017     heat   \n",
       "4     00cfe51c25b825270bc4509dfadca8d4f50a4e99   dec  2017     heat   \n",
       "...                                        ...   ...   ...      ...   \n",
       "2300  ff635b7556b71ecf409661242fa988bf1361c850   dec  2020     heat   \n",
       "2301  ff635b7556b71ecf409661242fa988bf1361c850   dec  2020     heat   \n",
       "2302  ffacf5dda3b0257b3891ad670f2d1f42187e2277   dec  2020     heat   \n",
       "2303  ffb88d949cb4d05658d8445b5cc64eb853aea426   dec  2020     heat   \n",
       "2304  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   dec  2020     auto   \n",
       "\n",
       "     CalendarEvent        City  Temperature_ctrl_ave  \\\n",
       "0             auto   Millstone            674.580038   \n",
       "1             hold   Millstone            695.525301   \n",
       "2             auto   Ridgewood            630.330435   \n",
       "3             hold   Ridgewood            653.267806   \n",
       "4             hold     Medford            620.880000   \n",
       "...            ...         ...                   ...   \n",
       "2300          auto      newark            680.130178   \n",
       "2301          hold      newark            684.165545   \n",
       "2302          auto   Oak Ridge            732.026316   \n",
       "2303          hold  Livingston            713.411765   \n",
       "2304          hold     Maywood            705.243827   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      674.732580                   673.749529   \n",
       "1                      700.939759                   700.679518   \n",
       "2                      650.268478                   630.450000   \n",
       "3                      661.336182                   638.943020   \n",
       "4                      651.400000                   620.280000   \n",
       "...                           ...                          ...   \n",
       "2300                   692.143984                   688.765286   \n",
       "2301                   689.464334                   685.562584   \n",
       "2302                   727.631579                   724.500000   \n",
       "2303                   720.000000                   720.000000   \n",
       "2304                   739.351852                   708.299383   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     0.0             False         False            False  \n",
       "3                     0.0             False         False            False  \n",
       "4                     0.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "2300                120.0             False         False            False  \n",
       "2301                120.0             False         False            False  \n",
       "2302                 99.0              True         False            False  \n",
       "2303                  0.0             False         False            False  \n",
       "2304                 90.0             False         False            False  \n",
       "\n",
       "[8669 rows x 13 columns]"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_dec = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"data/day/NJ/dec/\" + file)\n",
    "    NJ_dec = pd.concat([NJ_dec, df])\n",
    "    \n",
    "NJ_dec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_dec.to_csv(\"Scraper_Output/State_Month_Day/NJ/NJ_dec.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Combine state CSV Files \n",
    "1. Read in files in folders for each state\n",
    "2. Export as combined CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create variable for files in directory\n",
    "files = [f for f in os.listdir(\"Scraper_Output/State_Month_Day/NJ/\") if f.endswith(\".csv\")]\n",
    "\n",
    "# files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Identifier</th>\n",
       "      <th>Month</th>\n",
       "      <th>Year</th>\n",
       "      <th>HvacMode</th>\n",
       "      <th>CalendarEvent</th>\n",
       "      <th>City</th>\n",
       "      <th>Temperature_ctrl_ave</th>\n",
       "      <th>TemperatureExpectedCool_ave</th>\n",
       "      <th>TemperatureExpectedHeat_ave</th>\n",
       "      <th>Age_of_Home__years_</th>\n",
       "      <th>allowCompWithAux</th>\n",
       "      <th>Has_Electric</th>\n",
       "      <th>Has_a_Heat_Pump</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>775.569975</td>\n",
       "      <td>774.788804</td>\n",
       "      <td>668.452926</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>004f71a0c52a570bac8cebc13392ba54fc98da79</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Millstone</td>\n",
       "      <td>773.368932</td>\n",
       "      <td>767.621359</td>\n",
       "      <td>767.317152</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>auto</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>709.096579</td>\n",
       "      <td>707.939638</td>\n",
       "      <td>657.931590</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00fffc77256e65d8b7e1657b30e446dd9fa32eb5</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Hillsborough</td>\n",
       "      <td>675.947368</td>\n",
       "      <td>675.410526</td>\n",
       "      <td>624.842105</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0129f1aa4f23beac88b2e4bd022213101f2cac8f</td>\n",
       "      <td>aug</td>\n",
       "      <td>2017</td>\n",
       "      <td>cool</td>\n",
       "      <td>auto</td>\n",
       "      <td>Haddon Township</td>\n",
       "      <td>619.096234</td>\n",
       "      <td>638.418410</td>\n",
       "      <td>625.585774</td>\n",
       "      <td>5.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9668</th>\n",
       "      <td>fe66c5051b7e13329c0c52a696a10794fa5efd7d</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Swedesboro</td>\n",
       "      <td>730.272727</td>\n",
       "      <td>721.806061</td>\n",
       "      <td>720.733333</td>\n",
       "      <td>10.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9669</th>\n",
       "      <td>feada91765d02a17397b41f77689a4129b6b57d1</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Egg Harbor Township</td>\n",
       "      <td>711.272727</td>\n",
       "      <td>685.303030</td>\n",
       "      <td>685.363636</td>\n",
       "      <td>19.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9670</th>\n",
       "      <td>ff635b7556b71ecf409661242fa988bf1361c850</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>newark</td>\n",
       "      <td>751.298401</td>\n",
       "      <td>760.071048</td>\n",
       "      <td>759.863233</td>\n",
       "      <td>120.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9671</th>\n",
       "      <td>ffb88d949cb4d05658d8445b5cc64eb853aea426</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>cool</td>\n",
       "      <td>hold</td>\n",
       "      <td>Livingston</td>\n",
       "      <td>774.692308</td>\n",
       "      <td>768.307692</td>\n",
       "      <td>768.307692</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9672</th>\n",
       "      <td>ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af</td>\n",
       "      <td>jun</td>\n",
       "      <td>2021</td>\n",
       "      <td>auto</td>\n",
       "      <td>hold</td>\n",
       "      <td>Maywood</td>\n",
       "      <td>716.130869</td>\n",
       "      <td>715.254745</td>\n",
       "      <td>694.575425</td>\n",
       "      <td>90.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>51212 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    Identifier Month  Year HvacMode  \\\n",
       "0     004f71a0c52a570bac8cebc13392ba54fc98da79   aug  2017     cool   \n",
       "1     004f71a0c52a570bac8cebc13392ba54fc98da79   aug  2017     cool   \n",
       "2     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   aug  2017     auto   \n",
       "3     00fffc77256e65d8b7e1657b30e446dd9fa32eb5   aug  2017     auto   \n",
       "4     0129f1aa4f23beac88b2e4bd022213101f2cac8f   aug  2017     cool   \n",
       "...                                        ...   ...   ...      ...   \n",
       "9668  fe66c5051b7e13329c0c52a696a10794fa5efd7d   jun  2021     cool   \n",
       "9669  feada91765d02a17397b41f77689a4129b6b57d1   jun  2021     cool   \n",
       "9670  ff635b7556b71ecf409661242fa988bf1361c850   jun  2021     cool   \n",
       "9671  ffb88d949cb4d05658d8445b5cc64eb853aea426   jun  2021     cool   \n",
       "9672  ffbadc5a30ee14b1d5c4e5efca0326f5b7a877af   jun  2021     auto   \n",
       "\n",
       "     CalendarEvent                 City  Temperature_ctrl_ave  \\\n",
       "0             auto            Millstone            775.569975   \n",
       "1             hold            Millstone            773.368932   \n",
       "2             auto         Hillsborough            709.096579   \n",
       "3             hold         Hillsborough            675.947368   \n",
       "4             auto      Haddon Township            619.096234   \n",
       "...            ...                  ...                   ...   \n",
       "9668          hold           Swedesboro            730.272727   \n",
       "9669          hold  Egg Harbor Township            711.272727   \n",
       "9670          hold               newark            751.298401   \n",
       "9671          hold           Livingston            774.692308   \n",
       "9672          hold              Maywood            716.130869   \n",
       "\n",
       "      TemperatureExpectedCool_ave  TemperatureExpectedHeat_ave  \\\n",
       "0                      774.788804                   668.452926   \n",
       "1                      767.621359                   767.317152   \n",
       "2                      707.939638                   657.931590   \n",
       "3                      675.410526                   624.842105   \n",
       "4                      638.418410                   625.585774   \n",
       "...                           ...                          ...   \n",
       "9668                   721.806061                   720.733333   \n",
       "9669                   685.303030                   685.363636   \n",
       "9670                   760.071048                   759.863233   \n",
       "9671                   768.307692                   768.307692   \n",
       "9672                   715.254745                   694.575425   \n",
       "\n",
       "      Age_of_Home__years_  allowCompWithAux  Has_Electric  Has_a_Heat_Pump  \n",
       "0                     0.0             False         False            False  \n",
       "1                     0.0             False         False            False  \n",
       "2                     5.0             False         False            False  \n",
       "3                     5.0             False         False            False  \n",
       "4                     5.0             False         False            False  \n",
       "...                   ...               ...           ...              ...  \n",
       "9668                 10.0              True         False            False  \n",
       "9669                 19.0             False         False             True  \n",
       "9670                120.0             False         False            False  \n",
       "9671                  0.0             False         False            False  \n",
       "9672                 90.0             False         False            False  \n",
       "\n",
       "[51212 rows x 13 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# https://stackoverflow.com/questions/63886787/how-to-create-a-dataframe-from-multiple-csv-files\n",
    "# Initialize an empty dataframe\n",
    "NJ_all = pd.DataFrame()\n",
    "\n",
    "# Iterate through files and contents, then concatenate the data into the dataframe\n",
    "for file in files:\n",
    "    df = pd.read_csv(\"Scraper_Output/State_Month_Day/NJ/\" + file)\n",
    "    NJ_all = pd.concat([NJ_all, df])\n",
    "    \n",
    "NJ_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [],
   "source": [
    "NJ_all.to_csv(\"Scraper_Output/State_Month_Day/NJ_all_day.csv\", header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique jan_2017: ['NJ']\n",
      "Unique jan_2018: ['NJ']\n",
      "Unique jan_2019: ['NJ']\n",
      "Unique jan_2020: ['NJ']\n",
      "Unique jan_2021: ['NJ']\n",
      "Unique feb_2017: ['NJ']\n",
      "Unique feb_2018: ['NJ']\n",
      "Unique feb_2019: ['NJ']\n",
      "Unique feb_2020: ['NJ']\n",
      "Unique feb_2021: ['NJ']\n",
      "Unique jun_2017: ['NJ']\n",
      "Unique jun_2018: ['NJ']\n",
      "Unique jun_2019: ['NJ']\n",
      "Unique jun_2020: ['NJ']\n",
      "Unique jun_2021: ['NJ']\n",
      "Unique jul_2017: ['NJ']\n",
      "Unique jul_2018: ['NJ']\n",
      "Unique jul_2019: ['NJ']\n",
      "Unique jul_2020: ['NJ']\n",
      "Unique jul_2021: ['NJ']\n",
      "Unique aug_2017: ['NJ']\n",
      "Unique aug_2018: ['NJ']\n",
      "Unique aug_2019: ['NJ']\n",
      "Unique aug_2020: ['NJ']\n",
      "Unique dec_2017: ['NJ']\n",
      "Unique dec_2018: ['NJ']\n",
      "Unique dec_2019: ['NJ']\n",
      "Unique dec_2020: ['NJ']\n"
     ]
    }
   ],
   "source": [
    "# Datacheck to mNJe sure state was selected correctly in BQ sql queries\n",
    "\n",
    "print(f\"Unique jan_2017: {jan_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2018: {jan_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2019: {jan_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2020: {jan_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jan_2021: {jan_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2017: {feb_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2018: {feb_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2019: {feb_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2020: {feb_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique feb_2021: {feb_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2017: {jun_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2018: {jun_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2019: {jun_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2020: {jun_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jun_2021: {jun_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2017: {jul_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2018: {jul_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2019: {jul_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2020: {jul_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique jul_2021: {jul_2021['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2017: {aug_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2018: {aug_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2019: {aug_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique aug_2020: {aug_2020['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2017: {dec_2017['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2018: {dec_2018['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2019: {dec_2019['ProvinceState'].unique()}\")\n",
    "print(f\"Unique dec_2020: {dec_2020['ProvinceState'].unique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}